Skip to content

s3: support x-amz-meta-mode for symlinks (s3fs-fuse interop) #130

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mkviatkovskii wants to merge 1 commit into
base: main
Choose a base branch
from
Open

s3: support x-amz-meta-mode for symlinks (s3fs-fuse interop) #130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 51 additions & 1 deletion go/s3/s3.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"strconv"
"strings"
"sync"
"syscall"
"time"
"xtx/ternfs/client"
"xtx/ternfs/core/bufpool"
Expand Down Expand Up @@ -213,6 +214,53 @@ func FileEtag(id msgs.InodeId) string {
return fmt.Sprintf(`"%x"`, uint64(id))
}

// Modes echoed back on GET/HEAD/?attributes. TernFS has no per-inode
// permission storage, so we emit fixed sensible defaults whose only
// load-bearing bits are the file-type bits.
const (
s3DefaultDirMode = syscall.S_IFDIR | 0o755
s3DefaultFileMode = syscall.S_IFREG | 0o644
s3SymlinkMode = syscall.S_IFLNK | 0o777
)

// parseAmzMetaMode reads the s3fs-fuse "x-amz-meta-mode" header and returns
// the TernFS inode type implied by it: msgs.SYMLINK when the type bits are
// S_IFLNK, msgs.FILE otherwise (including when the header is absent or
// unparseable — we don't error on a malformed header). The header value is
// parsed with base 0 (decimal, octal "0...", or hex "0x...") to match
// s3fs-fuse's strtoll-with-base-0 reader.
func parseAmzMetaMode(r *http.Request) msgs.InodeType {
v := r.Header.Get("X-Amz-Meta-Mode")
if v == "" {
return msgs.FILE
}
mode, err := strconv.ParseUint(strings.TrimSpace(v), 0, 32)
if err != nil {
return msgs.FILE
}
if mode&syscall.S_IFMT == syscall.S_IFLNK {
return msgs.SYMLINK
}
return msgs.FILE
}

// emitAmzMetaMode writes a fixed x-amz-meta-mode header for the given inode
// type so s3fs-fuse can recognize the object's POSIX type on read. Without
// this header s3fs falls back to mode 0640/0750 and would lose the symlink
// type bit on round-trip.
func emitAmzMetaMode(w http.ResponseWriter, typ msgs.InodeType) {
var mode uint32
switch typ {
case msgs.SYMLINK:
mode = s3SymlinkMode
case msgs.DIRECTORY:
mode = s3DefaultDirMode
default:
mode = s3DefaultFileMode
}
w.Header().Set("X-Amz-Meta-Mode", strconv.FormatUint(uint64(mode), 10))
}

// parseRange parses the HTTP Range header.
func parseRange(rangeHeader string, totalSize int64) (start, length int64, err error) {
if !strings.HasPrefix(rangeHeader, "bytes=") {
Expand Down Expand Up @@ -495,6 +543,7 @@ func (s *S3Server) handleGetObject(ctx context.Context, w http.ResponseWriter, r
w.Header().Set("Last-Modified", lastModified.Time().Format(http.TimeFormat))
w.Header().Set("Accept-Ranges", "bytes")
w.Header().Set("Content-Type", "application/octet-stream")
emitAmzMetaMode(w, dentry.TargetId.Type())

if rangeHeader == "" {
w.Header().Set("Content-Length", strconv.FormatInt(int64(size), 10))
Expand Down Expand Up @@ -561,6 +610,7 @@ func (s *S3Server) handleGetObjectAttributes(ctx context.Context, w http.Respons
w.Header().Set("x-amz-object-attributes-last-modified", lastModified.Time().UTC().Format(http.TimeFormat))
w.Header().Set("x-amz-object-attributes-object-size", strconv.FormatInt(int64(size), 10))
w.Header().Set("x-amz-storage-class", "STANDARD")
emitAmzMetaMode(w, inode.Type())
w.WriteHeader(http.StatusOK)
return nil
}
Expand Down Expand Up @@ -723,7 +773,7 @@ func (s *S3Server) handlePutObject(ctx context.Context, w http.ResponseWriter, r
}
} else {
fileResp := msgs.ConstructFileResp{}
if err := s.c.ShardRequest(s.log, ownerId.Shard(), &msgs.ConstructFileReq{Type: msgs.FILE}, &fileResp); err != nil {
if err := s.c.ShardRequest(s.log, ownerId.Shard(), &msgs.ConstructFileReq{Type: parseAmzMetaMode(r)}, &fileResp); err != nil {
return fmt.Errorf("failed to construct file: %w", err)
}
fileId := fileResp.Id
Expand Down
117 changes: 117 additions & 0 deletions go/s3/s3_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
// Copyright 2025 XTX Markets Technologies Limited
//
// SPDX-License-Identifier: GPL-2.0-or-later

package s3

import (
"net/http"
"net/http/httptest"
"testing"
"xtx/ternfs/core/assert"
"xtx/ternfs/msgs"
)

func TestParseAmzMetaMode(t *testing.T) {
cases := []struct {
name string
header string // empty means header not set at all
want msgs.InodeType
}{
// s3fs-fuse emits decimal of full st_mode.
{"s3fs symlink (decimal 41471)", "41471", msgs.SYMLINK},
{"s3fs regular file 0644 (decimal 33188)", "33188", msgs.FILE},
{"s3fs directory 0755 (decimal 16877)", "16877", msgs.FILE},

// strconv.ParseUint with base=0 must accept octal and hex too,
// matching s3fs's strtoll(base=0) reader path.
{"octal symlink (0120777)", "0120777", msgs.SYMLINK},
{"octal regular file (0100644)", "0100644", msgs.FILE},
{"hex symlink (0xa1ff)", "0xa1ff", msgs.SYMLINK},
{"hex regular file (0x81a4)", "0x81a4", msgs.FILE},

// Setuid/setgid/sticky bits MUST NOT be misread as a type bit.
// 36388 = S_IFREG | S_ISUID | 0644.
{"setuid regular file (36388)", "36388", msgs.FILE},
// 17407 = S_IFDIR | S_ISVTX | 0777 — sticky-bit dir.
{"sticky directory (17407)", "17407", msgs.FILE},

// Type bits other than S_IFLNK map to a regular file.
{"FIFO 0010644", "0010644", msgs.FILE},
{"socket 0140644", "0140644", msgs.FILE},
{"char dev 0020644", "0020644", msgs.FILE},
{"block dev 0060644", "0060644", msgs.FILE},

// Tolerant fallback: missing or garbage header → regular file, no error.
{"absent header", "", msgs.FILE},
{"empty value", "", msgs.FILE},
{"garbage", "not-a-number", msgs.FILE},
{"trailing garbage", "41471xyz", msgs.FILE},

// Whitespace tolerance — clients sometimes pad headers.
{"leading and trailing space", " 41471 ", msgs.SYMLINK},

// Edge: zero is parseable but its type bits aren't S_IFLNK.
{"zero", "0", msgs.FILE},
}

for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
r := httptest.NewRequest(http.MethodPut, "/bucket/key", nil)
if tc.header != "" {
r.Header.Set("X-Amz-Meta-Mode", tc.header)
}
assert.Equal(t, tc.want, parseAmzMetaMode(r))
})
}
}

func TestParseAmzMetaModeIsCaseInsensitive(t *testing.T) {
// AWS canonicalizes user-metadata header names to lowercase; s3fs always
// emits lowercase. Ensure we accept any casing of "x-amz-meta-mode".
for _, headerName := range []string{
"x-amz-meta-mode",
"X-Amz-Meta-Mode",
"X-AMZ-META-MODE",
"X-aMz-MeTa-MoDe",
} {
t.Run(headerName, func(t *testing.T) {
r := httptest.NewRequest(http.MethodPut, "/bucket/key", nil)
r.Header.Set(headerName, "41471")
assert.Equal(t, msgs.SYMLINK, parseAmzMetaMode(r))
})
}
}

func TestEmitAmzMetaMode(t *testing.T) {
cases := []struct {
name string
typ msgs.InodeType
want string
}{
{"symlink", msgs.SYMLINK, "41471"}, // S_IFLNK | 0777
{"directory", msgs.DIRECTORY, "16877"}, // S_IFDIR | 0755
{"regular file", msgs.FILE, "33188"}, // S_IFREG | 0644
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
rec := httptest.NewRecorder()
emitAmzMetaMode(rec, tc.typ)
assert.Equal(t, tc.want, rec.Header().Get("X-Amz-Meta-Mode"))
})
}
}

// Round-trip: anything we emit for a SYMLINK must be parsed back as a SYMLINK.
// This guards against future edits where someone adjusts only one side
// (e.g. switches the constants to a different value or base) and silently
// breaks s3fs-fuse interop.
func TestSymlinkModeRoundTrip(t *testing.T) {
rec := httptest.NewRecorder()
emitAmzMetaMode(rec, msgs.SYMLINK)
emitted := rec.Header().Get("X-Amz-Meta-Mode")

r := httptest.NewRequest(http.MethodPut, "/bucket/key", nil)
r.Header.Set("X-Amz-Meta-Mode", emitted)
assert.Equal(t, msgs.SYMLINK, parseAmzMetaMode(r))
}