diff --git a/storage/drivers/chown_unix.go b/storage/drivers/chown_unix.go
index e4567ecec9..eec3db711a 100644
--- a/storage/drivers/chown_unix.go
+++ b/storage/drivers/chown_unix.go
@@ -19,7 +19,9 @@ type inode struct {
 }
 
 type platformChowner struct {
-	mutex  sync.Mutex
+	mutex sync.Mutex
+	// inodes to path map, so the chowner can reconstruct hard links
+	// To save memory only elements that are known to be hard links are added.
 	inodes map[inode]string
 }
 
@@ -40,16 +42,22 @@ func (c *platformChowner) LChown(path string, info os.FileInfo, toHost, toContai
 		Ino: st.Ino,
 	}
 
+	// Directories are assumed to never have hard links, they cannot be linked on Linux and the BSDs.
+	// Macos is the only one we know of that can but does not use the code here due "!darwin" build tag.
+	isHardLink := !info.IsDir() && st.Nlink > 1
+
 	c.mutex.Lock()
 
 	oldTarget, found := c.inodes[i]
-	if !found {
+	// Only add entries when we know hard links exists to safe memory allocations and
+	// the total map size. Otherwise we risk going out of memory for very large directory trees.
+	if !found && isHardLink {
 		c.inodes[i] = path
 	}
 
 	// If we are dealing with a file with multiple links then keep the lock until the file is
 	// chowned to avoid a race where we link to the old version if the file is copied up.
-	if found || st.Nlink > 1 {
+	if found || isHardLink {
 		defer c.mutex.Unlock()
 	} else {
 		c.mutex.Unlock()
diff --git a/storage/drivers/chown_unix_test.go b/storage/drivers/chown_unix_test.go
new file mode 100644
index 0000000000..b00a0e11b4
--- /dev/null
+++ b/storage/drivers/chown_unix_test.go
@@ -0,0 +1,69 @@
+//go:build !windows && !darwin
+
+package graphdriver
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"go.podman.io/storage/pkg/idtools"
+)
+
+func fillTestFiles(b *testing.B, path string, amount int) {
+	dirCount := 0
+	dir := path
+	for i := range amount {
+		if i%256 == 0 {
+			dir = filepath.Join(path, "dir"+strconv.Itoa(dirCount))
+			err := os.Mkdir(dir, 0o700)
+			require.NoError(b, err)
+			dirCount++
+		}
+		f, err := os.Create(filepath.Join(dir, strconv.Itoa(i)))
+		require.NoError(b, err)
+		f.Close()
+	}
+}
+
+func Benchmark_LChown(b *testing.B) {
+	uid := os.Getuid()
+	gid := os.Getgid()
+	ids := idtools.NewIDMappingsFromMaps([]idtools.IDMap{
+		{
+			ContainerID: uid,
+			HostID:      uid,
+			Size:        1,
+		},
+	}, []idtools.IDMap{
+		{
+			ContainerID: gid,
+			HostID:      gid,
+			Size:        1,
+		},
+	})
+
+	for _, amount := range []int{1, 10, 100, 1_000, 10_000, 100_000} {
+		b.Run(strconv.Itoa(amount), func(b *testing.B) {
+			path := b.TempDir()
+			fillTestFiles(b, path, amount)
+			for b.Loop() {
+				chowner := newLChowner()
+				var chown fs.WalkDirFunc = func(path string, d fs.DirEntry, _ error) error {
+					info, err := d.Info()
+					if err != nil {
+						return err
+					}
+					return chowner.LChown(path, info, ids, nil)
+				}
+				err := filepath.WalkDir(path, chown)
+				require.NoError(b, err)
+				// log as custom metric the number of elements in the inodes map to show the improvement best
+				b.ReportMetric(float64(len(chowner.inodes)), "inodes_count/op")
+			}
+		})
+	}
+}