diff --git a/go.mod b/go.mod
index 8d4af66addd..1e4755ef50e 100644
--- a/go.mod
+++ b/go.mod
@@ -40,7 +40,7 @@ require (
 	github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a
 	github.com/twpayne/go-pinentry/v3 v3.0.1
 	github.com/twpayne/go-shell v0.4.0
-	github.com/twpayne/go-vfs/v5 v5.0.3
+	github.com/twpayne/go-vfs/v5 v5.0.4
 	github.com/twpayne/go-xdg/v6 v6.1.2
 	github.com/ulikunitz/xz v0.5.11
 	github.com/zalando/go-keyring v0.2.4
diff --git a/go.sum b/go.sum
index a7ff7ad845c..2ed7cb93888 100644
--- a/go.sum
+++ b/go.sum
@@ -423,8 +423,8 @@ github.com/twpayne/go-shell v0.4.0 h1:RAAMbjEj7mcwDdwC7SiFHGUKR+WDAURU6mnyd3r2p2
 github.com/twpayne/go-shell v0.4.0/go.mod h1:MP3aUA0TQ3IGoJc15ahjb+7A7wZH4NeGrvLZ/aFQsHc=
 github.com/twpayne/go-vfs/v4 v4.3.0 h1:rTqFzzOQ/6ESKTSiwVubHlCBedJDOhQyVSnw8rQNZhU=
 github.com/twpayne/go-vfs/v4 v4.3.0/go.mod h1:tq2UVhnUepesc0lSnPJH/jQ8HruGhzwZe2r5kDFpEIw=
-github.com/twpayne/go-vfs/v5 v5.0.3 h1:9jeacSMvwh8sSLCuS4Ay0QmyAv1ToG1ITeQn6+Zsq1k=
-github.com/twpayne/go-vfs/v5 v5.0.3/go.mod h1:zTPFJUbgsEMFNSWnWQlLq9wh4AN83edZzx3VXbxrS1w=
+github.com/twpayne/go-vfs/v5 v5.0.4 h1:/ne3h+rW7f5YOyOFguz+3ztfUwzOLR0Vts3y0mMAitg=
+github.com/twpayne/go-vfs/v5 v5.0.4/go.mod h1:zTPFJUbgsEMFNSWnWQlLq9wh4AN83edZzx3VXbxrS1w=
 github.com/twpayne/go-xdg/v6 v6.1.2 h1:KbfCsAP4bBR5+dzfTIh/M9onOPCSqlYsIER79IKwt+s=
 github.com/twpayne/go-xdg/v6 v6.1.2/go.mod h1:BFHclQaEPLq3jRRYjf1PdFzUEvAfPeLjNymIO/7/7o4=
 github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
diff --git a/internal/cmd/addcmd.go b/internal/cmd/addcmd.go
index 480518416a5..f94028a6f8a 100644
--- a/internal/cmd/addcmd.go
+++ b/internal/cmd/addcmd.go
@@ -164,8 +164,9 @@ func (c *Config) defaultReplaceFunc(
 
 func (c *Config) runAddCmd(cmd *cobra.Command, args []string, sourceState *chezmoi.SourceState) error {
 	destAbsPathInfos, err := c.destAbsPathInfos(sourceState, args, destAbsPathInfosOptions{
-		follow:    c.Mode == chezmoi.ModeSymlink || c.Add.follow,
-		recursive: c.Add.recursive,
+		follow:       c.Mode == chezmoi.ModeSymlink || c.Add.follow,
+		onIgnoreFunc: c.defaultOnIgnoreFunc,
+		recursive:    c.Add.recursive,
 	})
 	if err != nil {
 		return err
diff --git a/internal/cmd/config.go b/internal/cmd/config.go
index 9647df26923..96daea78b4f 100644
--- a/internal/cmd/config.go
+++ b/internal/cmd/config.go
@@ -1071,6 +1071,7 @@ func (c *Config) defaultSourceDir(fileSystem vfs.Stater, bds *xdg.BaseDirectoryS
 type destAbsPathInfosOptions struct {
 	follow         bool
 	ignoreNotExist bool
+	onIgnoreFunc   func(chezmoi.RelPath)
 	recursive      bool
 }
 
@@ -1089,9 +1090,14 @@ func (c *Config) destAbsPathInfos(
 		if err != nil {
 			return nil, err
 		}
-		if _, err := c.targetRelPath(destAbsPath); err != nil {
+		targetRelPath, err := c.targetRelPath(destAbsPath)
+		if err != nil {
 			return nil, err
 		}
+		if sourceState.Ignore(targetRelPath) {
+			options.onIgnoreFunc(targetRelPath)
+			continue
+		}
 		if options.recursive {
 			walkFunc := func(destAbsPath chezmoi.AbsPath, fileInfo fs.FileInfo, err error) error {
 				switch {
@@ -1100,12 +1106,26 @@ func (c *Config) destAbsPathInfos(
 				case err != nil:
 					return err
 				}
+
+				targetRelPath, err := c.targetRelPath(destAbsPath)
+				if err != nil {
+					return err
+				}
+				if sourceState.Ignore(targetRelPath) {
+					options.onIgnoreFunc(targetRelPath)
+					if fileInfo.IsDir() {
+						return fs.SkipDir
+					}
+					return nil
+				}
+
 				if options.follow && fileInfo.Mode().Type() == fs.ModeSymlink {
 					fileInfo, err = c.destSystem.Stat(destAbsPath)
 					if err != nil {
 						return err
 					}
 				}
+
 				return sourceState.AddDestAbsPathInfos(destAbsPathInfos, c.destSystem, destAbsPath, fileInfo)
 			}
 			if err := chezmoi.Walk(c.destSystem, destAbsPath, walkFunc); err != nil {
diff --git a/internal/cmd/mackupcmd_darwin.go b/internal/cmd/mackupcmd_darwin.go
index 0e0ed986a82..a3aa708ef16 100644
--- a/internal/cmd/mackupcmd_darwin.go
+++ b/internal/cmd/mackupcmd_darwin.go
@@ -97,6 +97,7 @@ func (c *Config) runMackupAddCmd(cmd *cobra.Command, args []string, sourceState
 	destAbsPathInfos, err := c.destAbsPathInfos(sourceState, addArgs, destAbsPathInfosOptions{
 		follow:         c.Add.follow,
 		ignoreNotExist: true,
+		onIgnoreFunc:   c.defaultOnIgnoreFunc,
 		recursive:      c.Add.recursive,
 	})
 	if err != nil {
diff --git a/internal/cmd/testdata/scripts/add.txtar b/internal/cmd/testdata/scripts/add.txtar
index b931ac09d1c..6b8699b2f4a 100644
--- a/internal/cmd/testdata/scripts/add.txtar
+++ b/internal/cmd/testdata/scripts/add.txtar
@@ -59,7 +59,7 @@ chhome home3/user
 # test that chezmoi add respects .chezmoiignore
 exec chezmoi add $HOME${/}.dir
 exists $CHEZMOISOURCEDIR/dot_dir/file
-stderr 'warning: ignoring'
+stderr 'warning: ignoring .dir/ignore'
 ! exists $CHEZMOISOURCEDIR/dot_dir/ignore
 
 chhome home4/user
diff --git a/internal/cmd/testdata/scripts/issue3630.txtar b/internal/cmd/testdata/scripts/issue3630.txtar
new file mode 100644
index 00000000000..5ef9ffa8e1c
--- /dev/null
+++ b/internal/cmd/testdata/scripts/issue3630.txtar
@@ -0,0 +1,14 @@
+# test that chezmoi add does not traverse into ignored directories
+chmod 000 $HOME/.dir/private
+exec chezmoi add $HOME${/}.dir
+stderr 'warning: ignoring .dir/private'
+cmp $CHEZMOISOURCEDIR/dot_dir/public/file golden/file
+
+-- golden/file --
+# contents of .dir/public/file
+-- home/user/.dir/private/file --
+# contents of .dir/private/file
+-- home/user/.dir/public/file --
+# contents of .dir/public/file
+-- home/user/.local/share/chezmoi/.chezmoiignore --
+.dir/private