Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions adk/filesystem/backend_inmemory.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ package filesystem
import (
"context"
"fmt"
"path/filepath"
pathpkg "path"
"regexp"
"strings"
"sync"
Expand Down Expand Up @@ -72,7 +72,7 @@ func (b *InMemoryBackend) LsInfo(ctx context.Context, req *LsInfoRequest) ([]Fil
// The path itself is a file
if !seen[normalizedFilePath] {
result = append(result, FileInfo{
Path: filepath.Base(normalizedFilePath),
Path: pathpkg.Base(normalizedFilePath),
IsDir: false,
Size: int64(len(entry.content)),
ModifiedAt: entry.modifiedAt.Format(time.RFC3339Nano),
Expand Down Expand Up @@ -378,7 +378,7 @@ func (b *InMemoryBackend) filterByGlob(files []string, searchPath string, globPa
matchPath = strings.TrimPrefix(filePath, searchPath+"/")
}
} else {
matchPath = filepath.Base(filePath)
matchPath = pathpkg.Base(filePath)
}

matched, err := doublestar.Match(globPattern, matchPath)
Expand All @@ -397,7 +397,7 @@ func (b *InMemoryBackend) filterByFileType(files []string, fileType string) []st
var result []string

for _, filePath := range files {
ext := strings.TrimPrefix(filepath.Ext(filePath), ".")
ext := strings.TrimPrefix(pathpkg.Ext(filePath), ".")
if matchFileType(ext, fileType) {
result = append(result, filePath)
}
Expand Down Expand Up @@ -691,13 +691,14 @@ func normalizePath(path string) string {
if path == "" {
return "/"
}
path = strings.ReplaceAll(path, "\\", "/")

// Ensure path starts with "/"
if !strings.HasPrefix(path, "/") {
path = "/" + path
}

return filepath.Clean(path)
return pathpkg.Clean(path)
}

type grepCollector struct {
Expand Down
20 changes: 20 additions & 0 deletions adk/filesystem/backend_inmemory_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,26 @@ func TestInMemoryBackend_WriteAndRead(t *testing.T) {
}
}

func TestInMemoryBackend_NormalizePath_WindowsSeparator(t *testing.T) {
b := NewInMemoryBackend()
ctx := context.Background()

err := b.Write(ctx, &WriteRequest{
FilePath: "\\windows\\style\\file.txt",
Content: "hello",
})
if err != nil {
t.Fatalf("Write failed: %v", err)
}

out, err := b.Read(ctx, &ReadRequest{FilePath: "/windows/style/file.txt"})
if err != nil {
t.Fatalf("Read failed: %v", err)
}
if out.Content != "hello" {
t.Fatalf("expected %q, got %q", "hello", out.Content)
}
}
func TestInMemoryBackend_LsInfo(t *testing.T) {
backend := NewInMemoryBackend()
ctx := context.Background()
Expand Down
106 changes: 89 additions & 17 deletions adk/middlewares/filesystem/filesystem.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ const (

noFilesFound = "No files found"
noMatchesFound = "No matches found"

defaultReadFileLineLimit = 2000
defaultReadFileColumnLimit = 2000
)

// ToolConfig configures a filesystem tool
Expand Down Expand Up @@ -208,10 +211,15 @@ func NewMiddleware(ctx context.Context, config *Config) (adk.AgentMiddleware, er
}

if !config.WithoutLargeToolResultOffloading {
readFileToolName := ToolNameReadFile
if config.ReadFileToolConfig != nil && config.ReadFileToolConfig.Name != "" {
readFileToolName = config.ReadFileToolConfig.Name
}
m.WrapToolCall = newToolResultOffloading(ctx, &toolResultOffloadingConfig{
Backend: config.Backend,
TokenLimit: config.LargeToolResultOffloadingTokenLimit,
PathGenerator: config.LargeToolResultOffloadingPathGen,
ExcludeTools: []string{readFileToolName},
})
}

Expand Down Expand Up @@ -592,6 +600,12 @@ type readFileArgs struct {

// Limit is the number of lines to read.
Limit int `json:"limit" jsonschema:"description=The number of lines to read. Only provide if the file is too large to read at once."`

// ColumnOffset is the 1-based character column to start reading from within each returned line.
ColumnOffset int `json:"column_offset,omitempty" jsonschema:"description=The 1-based character column to start reading from within each returned line. Use with column_limit for very long single-line files. Values less than 1 are treated as 1."`

// ColumnLimit is the maximum number of characters to read from each returned line.
ColumnLimit int `json:"column_limit,omitempty" jsonschema:"description=The maximum number of characters to read from each returned line. Defaults to 2000 to keep very long single-line files readable. Set to -1 to disable column limiting."`
}

// multiModalReadFileArgs extends readFileArgs with PDF-specific parameters for MultiModalReadFileTool.
Expand All @@ -609,12 +623,7 @@ func newReadFileTool(fs filesystem.Backend, name string, desc string) (tool.Base
return nil, err
}
return utils.InferTool(toolName, d, func(ctx context.Context, input readFileArgs) (string, error) {
if input.Offset <= 0 {
input.Offset = 1
}
if input.Limit <= 0 {
input.Limit = 2000
}
normalizeReadFileArgs(&input)

fileCt, err := fs.Read(ctx, &filesystem.ReadRequest{
FilePath: input.FilePath,
Expand All @@ -628,26 +637,94 @@ func newReadFileTool(fs filesystem.Backend, name string, desc string) (tool.Base
return fmt.Sprintf("No content found at path: %s", input.FilePath), nil
}

return formatLineNumbers(fileCt.Content, input.Offset), nil
return formatLineNumbers(fileCt.Content, input.Offset, input.ColumnOffset, input.ColumnLimit), nil
})
}

func normalizeReadFileArgs(input *readFileArgs) {
if input.Offset <= 0 {
input.Offset = 1
}
if input.Limit <= 0 {
input.Limit = defaultReadFileLineLimit
}
if input.ColumnOffset <= 0 {
input.ColumnOffset = 1
}
if input.ColumnLimit == 0 {
input.ColumnLimit = defaultReadFileColumnLimit
}
}

// formatLineNumbers prefixes each line of content with a 1-based line number
// starting at startLine (e.g. " 1\tfoo"). startLine corresponds to the
// line number of the first line in content (usually ReadRequest.Offset).
func formatLineNumbers(content string, startLine int) string {
func formatLineNumbers(content string, startLine, columnOffset, columnLimit int) string {
lines := strings.Split(content, "\n")
var b strings.Builder
for i, line := range lines {
lineNo := startLine + i
slicedLine, note := sliceLineByColumns(line, lineNo, columnOffset, columnLimit)
if i < len(lines)-1 {
fmt.Fprintf(&b, "%6d\t%s\n", startLine+i, line)
fmt.Fprintf(&b, "%6d\t%s", lineNo, slicedLine)
if note != "" {
fmt.Fprintf(&b, "\n[%s]", note)
}
b.WriteByte('\n')
} else {
fmt.Fprintf(&b, "%6d\t%s", startLine+i, line)
fmt.Fprintf(&b, "%6d\t%s", lineNo, slicedLine)
if note != "" {
fmt.Fprintf(&b, "\n[%s]", note)
}
}
}
return b.String()
}

func sliceLineByColumns(line string, lineNo, columnOffset, columnLimit int) (string, string) {
if columnOffset <= 0 {
columnOffset = 1
}
if columnLimit == 0 {
columnLimit = defaultReadFileColumnLimit
}
if columnOffset == 1 && columnLimit < 0 {
return line, ""
}

runes := []rune(line)
totalColumns := len(runes)
if totalColumns == 0 {
if columnOffset == 1 {
return "", ""
}
return "", fmt.Sprintf("Line %d has no content at column_offset=%d.", lineNo, columnOffset)
}

start := columnOffset - 1
if start >= totalColumns {
return "", fmt.Sprintf("Line %d has %d columns; column_offset=%d is beyond the end.", lineNo, totalColumns, columnOffset)
}

end := totalColumns
if columnLimit > 0 && start+columnLimit < end {
end = start + columnLimit
}

segment := string(runes[start:end])
if start == 0 && end == totalColumns {
return segment, ""
}
if end < totalColumns {
nextLimit := columnLimit
if nextLimit < 0 {
nextLimit = defaultReadFileColumnLimit
}
return segment, fmt.Sprintf("Line %d truncated: showing columns %d-%d of %d. Use column_offset=%d and column_limit=%d to continue.", lineNo, start+1, end, totalColumns, end+1, nextLimit)
}
return segment, fmt.Sprintf("Line %d starts at column %d of %d.", lineNo, start+1, totalColumns)
}

const maxPagesPerRequest = 20

func validatePages(pages string) error {
Expand Down Expand Up @@ -697,12 +774,7 @@ func newMultiModalReadFileTool(fs filesystem.Backend, name string, desc string)
}

return utils.InferEnhancedTool(toolName, d, func(ctx context.Context, input multiModalReadFileArgs) (*schema.ToolResult, error) {
if input.Offset <= 0 {
input.Offset = 1
}
if input.Limit <= 0 {
input.Limit = 2000
}
normalizeReadFileArgs(&input.readFileArgs)

if input.Pages != "" {
if err := validatePages(input.Pages); err != nil {
Expand Down Expand Up @@ -779,7 +851,7 @@ func newMultiModalReadFileTool(fs filesystem.Backend, name string, desc string)
}

return &schema.ToolResult{
Parts: []schema.ToolOutputPart{{Type: schema.ToolPartTypeText, Text: formatLineNumbers(fileCt.Content, input.Offset)}},
Parts: []schema.ToolOutputPart{{Type: schema.ToolPartTypeText, Text: formatLineNumbers(fileCt.Content, input.Offset, input.ColumnOffset, input.ColumnLimit)}},
}, nil
})
}
Expand Down
89 changes: 88 additions & 1 deletion adk/middlewares/filesystem/filesystem_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,93 @@ func TestReadFileTool_DefaultLimit(t *testing.T) {
})
}

func TestReadFileTool_LongSingleLineColumnPagination(t *testing.T) {
backend := filesystem.NewInMemoryBackend()
longLine := strings.Repeat("a", 2500) + strings.Repeat("b", 500)
err := backend.Write(context.Background(), &filesystem.WriteRequest{
FilePath: "/long.txt",
Content: longLine,
})
assert.NoError(t, err)

readTool, err := newReadFileTool(backend, "", "")
assert.NoError(t, err)

result, err := invokeTool(t, readTool, `{"file_path": "/long.txt", "offset": 1, "limit": 1}`)
assert.NoError(t, err)
assert.Contains(t, result, fmt.Sprintf(" 1\t%s", strings.Repeat("a", 2000)))
assert.NotContains(t, result, strings.Repeat("a", 2001))
assert.Contains(t, result, "Line 1 truncated: showing columns 1-2000 of 3000")
assert.Contains(t, result, "column_offset=2001")

result, err = invokeTool(t, readTool, `{"file_path": "/long.txt", "offset": 1, "limit": 1, "column_offset": 2001, "column_limit": 500}`)
assert.NoError(t, err)
assert.Contains(t, result, fmt.Sprintf(" 1\t%s", strings.Repeat("a", 500)))
assert.Contains(t, result, "Line 1 truncated: showing columns 2001-2500 of 3000")
assert.Contains(t, result, "column_offset=2501")

result, err = invokeTool(t, readTool, `{"file_path": "/long.txt", "offset": 1, "limit": 1, "column_offset": 2501, "column_limit": 500}`)
assert.NoError(t, err)
assert.Equal(t, fmt.Sprintf(" 1\t%s\n[Line 1 starts at column 2501 of 3000.]", strings.Repeat("b", 500)), result)
}

func TestReadFileTool_ColumnPaginationEdgeCases(t *testing.T) {
backend := filesystem.NewInMemoryBackend()
assert.NoError(t, backend.Write(context.Background(), &filesystem.WriteRequest{
FilePath: "/file.txt",
Content: "aaaaaaaaaa",
}))

readTool, err := newReadFileTool(backend, "", "")
assert.NoError(t, err)

r, err := invokeTool(t, readTool, `{"file_path":"/file.txt", "offset":1, "limit":1, "column_limit":-1}`)
assert.NoError(t, err)
assert.Contains(t, r, "aaaaaaaaaa")
assert.NotContains(t, r, "truncated")
assert.NotContains(t, r, "starts at column")

r, err = invokeTool(t, readTool, `{"file_path":"/file.txt", "offset":1, "limit":1, "column_offset":50, "column_limit":10}`)
assert.NoError(t, err)
assert.Contains(t, r, "column_offset=50 is beyond the end")

r, err = invokeTool(t, readTool, `{"file_path":"/file.txt", "offset":1, "limit":1, "column_offset":5, "column_limit":100}`)
assert.NoError(t, err)
assert.Contains(t, r, fmt.Sprintf(" 1\t%s", "aaaaaa"))
assert.NotContains(t, r, "truncated")
assert.Contains(t, r, "Line 1 starts at column 5 of 10")
}

func TestReadFileTool_ColumnPaginationOnEmptyLine(t *testing.T) {
backend := filesystem.NewInMemoryBackend()
assert.NoError(t, backend.Write(context.Background(), &filesystem.WriteRequest{
FilePath: "/mixed.txt",
Content: "\nhello",
}))

readTool, err := newReadFileTool(backend, "", "")
assert.NoError(t, err)

r, err := invokeTool(t, readTool, `{"file_path":"/mixed.txt", "offset":1, "limit":1, "column_offset":3, "column_limit":10}`)
assert.NoError(t, err)
assert.Contains(t, r, "Line 1 has no content at column_offset=3")
}

func TestReadFileTool_ColumnOffsetBoundary(t *testing.T) {
backend := filesystem.NewInMemoryBackend()
assert.NoError(t, backend.Write(context.Background(), &filesystem.WriteRequest{
FilePath: "/b.txt",
Content: "abcdef",
}))

readTool, err := newReadFileTool(backend, "", "")
assert.NoError(t, err)

r, err := invokeTool(t, readTool, `{"file_path":"/b.txt", "offset":1, "limit":1, "column_offset":7, "column_limit":5}`)
assert.NoError(t, err)
assert.Contains(t, r, "column_offset=7 is beyond the end")
}

func TestWriteFileTool(t *testing.T) {
backend := setupTestBackend()
writeTool, err := newWriteFileTool(backend, "", "")
Expand Down Expand Up @@ -2502,7 +2589,7 @@ func TestMultiModalReadFileTool_SchemaContainsAllFields(t *testing.T) {
assert.NotNil(t, js)
assert.NotNil(t, js.Properties, "schema should have properties")

for _, field := range []string{"file_path", "offset", "limit", "pages"} {
for _, field := range []string{"file_path", "offset", "limit", "column_offset", "column_limit", "pages"} {
_, ok := js.Properties.Get(field)
assert.True(t, ok, "expected JSON schema to contain field %q, schema=%+v", field, js.Properties)
}
Expand Down
11 changes: 11 additions & 0 deletions adk/middlewares/filesystem/large_tool_result.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,18 @@ type toolResultOffloadingConfig struct {
Backend filesystem.Backend
TokenLimit int
PathGenerator func(ctx context.Context, input *compose.ToolInput) (string, error)
ExcludeTools []string
}

func newToolResultOffloading(ctx context.Context, config *toolResultOffloadingConfig) compose.ToolMiddleware {
offloading := &toolResultOffloading{
backend: config.Backend,
tokenLimit: config.TokenLimit,
pathGenerator: config.PathGenerator,
excludeTools: make(map[string]struct{}, len(config.ExcludeTools)),
}
for _, toolName := range config.ExcludeTools {
offloading.excludeTools[toolName] = struct{}{}
}

if offloading.tokenLimit == 0 {
Expand All @@ -66,6 +71,7 @@ type toolResultOffloading struct {
backend filesystem.Backend
tokenLimit int
pathGenerator func(ctx context.Context, input *compose.ToolInput) (string, error)
excludeTools map[string]struct{}
}

func (t *toolResultOffloading) invoke(endpoint compose.InvokableToolEndpoint) compose.InvokableToolEndpoint {
Expand Down Expand Up @@ -101,6 +107,11 @@ func (t *toolResultOffloading) stream(endpoint compose.StreamableToolEndpoint) c
}

func (t *toolResultOffloading) handleResult(ctx context.Context, result string, input *compose.ToolInput) (string, error) {
if input != nil {
if _, excluded := t.excludeTools[input.Name]; excluded {
return result, nil
}
}
if len(result) > t.tokenLimit*4 {
path, err := t.pathGenerator(ctx, input)
if err != nil {
Expand Down
Loading
Loading