Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .bazelversion
Original file line number Diff line number Diff line change
@@ -1 +1 @@
9.0.1
9.1.0
24 changes: 10 additions & 14 deletions MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -6,31 +6,31 @@ bazel_dep(name = "cel-spec", version = "0.25.1")
bazel_dep(name = "com_github_buildbarn_bb_remote_execution")
bazel_dep(name = "com_github_buildbarn_bb_storage")
bazel_dep(name = "com_github_buildbarn_go_xdr")
bazel_dep(name = "gazelle", version = "0.47.0")
bazel_dep(name = "jsonnet_go", version = "0.21.0")
bazel_dep(name = "gazelle", version = "0.51.0")
bazel_dep(name = "jsonnet_go", version = "0.22.0")
bazel_dep(name = "platforms", version = "1.0.0")
bazel_dep(name = "protobuf", version = "33.5")
bazel_dep(name = "rules_jsonnet", version = "0.7.2")
bazel_dep(name = "protobuf", version = "35.0")
bazel_dep(name = "rules_go", version = "0.60.0")
bazel_dep(name = "rules_jsonnet", version = "0.7.2")
bazel_dep(name = "rules_pkg", version = "1.2.0")
bazel_dep(name = "rules_proto", version = "7.1.0")
bazel_dep(name = "toolchains_llvm", version = "1.6.0")
bazel_dep(name = "toolchains_llvm", version = "1.7.0")

git_override(
module_name = "bazel_remote_apis",
commit = "715b73f3f9e4c70b22854e3cf8a927a9077dde1c",
commit = "becdd8f9ff811df88a22d3eadd6341753d51d167",
patches = ["//:patches/bazel_remote_apis/upstream-pr-378.diff"],
remote = "https://github.com/bazelbuild/remote-apis.git",
)

git_override(
module_name = "com_github_buildbarn_bb_remote_execution",
commit = "cd89f0554d18a1e8a0f3f4e3155b273fb2d3fff7",
commit = "ca3fedba831b0d922b9528b38c3daf50a058ae7f",
remote = "https://github.com/buildbarn/bb-remote-execution.git",
)

git_override(
module_name = "com_github_buildbarn_bb_storage",
commit = "dc342e1799d72bd927ffc0af1f429e7044118b07",
commit = "3991d6ebefb4cfc7c1956caaba5a28d1b2eca927",
remote = "https://github.com/buildbarn/bb-storage.git",
)

Expand All @@ -40,11 +40,6 @@ git_override(
remote = "https://github.com/buildbarn/go-xdr.git",
)

single_version_override(
module_name = "jsonnet_go",
patches = ["//:patches/jsonnet_go/bazel-9.diff"],
)

git_override(
module_name = "rules_antlr",
commit = "89a29cca479363a5aee53e203719510bdc6be6ff",
Expand All @@ -70,6 +65,7 @@ use_repo(
go_deps,
"cc_mvdan_gofumpt",
"com_github_bazelbuild_buildtools",
"com_github_bazelbuild_remote_apis",
"com_github_stretchr_testify",
"org_golang_google_genproto_googleapis_bytestream",
"org_golang_google_grpc",
Expand Down
1,294 changes: 717 additions & 577 deletions MODULE.bazel.lock

Large diffs are not rendered by default.

87 changes: 61 additions & 26 deletions cmd/bb_clientd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,8 @@ func main() {
util.DefaultErrorLogger,
time.Second,
30*time.Second,
maximumDelay.AsDuration())
maximumDelay.AsDuration(),
)
}

// Create the virtual file system.
Expand All @@ -124,28 +125,34 @@ func main() {
re_cas.NewBlobAccessDirectoryFetcher(
retryingContentAddressableStorage,
int(configuration.MaximumMessageSizeBytes),
configuration.MaximumTreeSizeBytes))
configuration.MaximumTreeSizeBytes,
),
)
if err != nil {
return util.StatusWrap(err, "Failed to create caching directory fetcher")
}
casFileFactory := re_vfs.NewResolvableHandleAllocatingCASFileFactory(
re_vfs.NewBlobAccessCASFileFactory(
context.Background(),
retryingContentAddressableStorage,
util.DefaultErrorLogger),
rootHandleAllocator.New())
util.DefaultErrorLogger,
),
rootHandleAllocator.New(),
)
decomposedCASDirectoryFactory := cd_vfs.NewDecomposedCASDirectoryFactory(
context.Background(),
casFileFactory,
directoryFetcher,
rootHandleAllocator.New(),
util.DefaultErrorLogger)
util.DefaultErrorLogger,
)
treeCASDirectoryFactory := cd_vfs.NewTreeCASDirectoryFactory(
context.Background(),
casFileFactory,
directoryFetcher,
rootHandleAllocator.New(),
util.DefaultErrorLogger)
util.DefaultErrorLogger,
)

// Factory function for per instance name "blobs" directories
// that give access to arbitrary files, directories and trees.
Expand All @@ -155,8 +162,10 @@ func main() {
context.Background(),
retryingContentAddressableStorage,
int(configuration.MaximumMessageSizeBytes),
util.DefaultErrorLogger),
rootHandleAllocator.New())
util.DefaultErrorLogger,
),
rootHandleAllocator.New(),
)
blobsDirectoryLookupFunc := func(instanceName digest.InstanceName) re_vfs.Directory {
handleAllocator := blobsDirectoryHandleAllocator.
New(re_vfs.ByteSliceID([]byte(instanceName.String()))).
Expand All @@ -182,32 +191,44 @@ func main() {
func(digest digest.Digest) (re_vfs.DirectoryChild, re_vfs.Status) {
f, s := commandFileFactory.LookupFile(digest)
return re_vfs.DirectoryChild{}.FromLeaf(f), s
}))),
},
)),
),
path.MustNewComponent("directory"): re_vfs.DirectoryChild{}.FromDirectory(
allocateHandle().AsStatelessDirectory(cd_vfs.NewDigestParsingDirectory(
digestFunction,
func(digest digest.Digest) (re_vfs.DirectoryChild, re_vfs.Status) {
return re_vfs.DirectoryChild{}.FromDirectory(decomposedCASDirectoryFactory.LookupDirectory(digest)), re_vfs.StatusOK
}))),
},
)),
),
path.MustNewComponent("executable"): re_vfs.DirectoryChild{}.FromDirectory(
allocateHandle().AsStatelessDirectory(cd_vfs.NewDigestParsingDirectory(
digestFunction,
func(digest digest.Digest) (re_vfs.DirectoryChild, re_vfs.Status) {
return re_vfs.DirectoryChild{}.FromLeaf(casFileFactory.LookupFile(digest, true, nil)), re_vfs.StatusOK
}))),
},
)),
),
path.MustNewComponent("file"): re_vfs.DirectoryChild{}.FromDirectory(
allocateHandle().AsStatelessDirectory(cd_vfs.NewDigestParsingDirectory(
digestFunction,
func(digest digest.Digest) (re_vfs.DirectoryChild, re_vfs.Status) {
return re_vfs.DirectoryChild{}.FromLeaf(casFileFactory.LookupFile(digest, false, nil)), re_vfs.StatusOK
}))),
},
)),
),
path.MustNewComponent("tree"): re_vfs.DirectoryChild{}.FromDirectory(
allocateHandle().AsStatelessDirectory(cd_vfs.NewDigestParsingDirectory(
digestFunction,
func(digest digest.Digest) (re_vfs.DirectoryChild, re_vfs.Status) {
return re_vfs.DirectoryChild{}.FromDirectory(treeCASDirectoryFactory.LookupDirectory(digest)), re_vfs.StatusOK
}))),
})))
},
)),
),
},
)),
)
}
return allocateHandle().AsStatelessDirectory(re_vfs.NewStaticDirectory(re_vfs.CaseSensitiveComponentNormalizer, blobsDirectoryContents))
}
Expand All @@ -216,8 +237,9 @@ func main() {
// Output Service allows Bazel to place its bazel-out/
// directories on a virtual file system, thereby
// allowing data to be loaded lazily.
defaultAttributesSetter := func(requested re_vfs.AttributesMask, attributes *re_vfs.Attributes) {}
symlinkFactory := re_vfs.NewHandleAllocatingSymlinkFactory(
re_vfs.BaseSymlinkFactory,
re_vfs.NewBaseSymlinkFactory(defaultAttributesSetter),
rootHandleAllocator.New(),
path.LocalFormat,
)
Expand All @@ -232,7 +254,8 @@ func main() {
outputPathFactory,
bareContentAddressableStorage,
util.DefaultErrorLogger,
semaphore.NewWeighted(concurrency))
semaphore.NewWeighted(concurrency),
)
}

// Enable persistent storage of bazel-out/ directories.
Expand All @@ -249,12 +272,15 @@ func main() {
outputpathpersistency.NewMaximumAgeStore(
outputpathpersistency.NewDirectoryBackedStore(
stateDirectory,
persistencyConfiguration.MaximumStateFileSizeBytes),
persistencyConfiguration.MaximumStateFileSizeBytes,
),
clock.SystemClock,
maximumStateFileAge.AsDuration()),
maximumStateFileAge.AsDuration(),
),
clock.SystemClock,
util.DefaultErrorLogger,
symlinkFactory)
symlinkFactory,
)
}

outputsDirectory := cd_vfs.NewBazelOutputServiceDirectory(
Expand All @@ -264,15 +290,15 @@ func main() {
retryingContentAddressableStorage,
directoryFetcher,
symlinkFactory,
configuration.MaximumTreeSizeBytes)
configuration.MaximumTreeSizeBytes,
)

// Construct the top-level directory of the virtual file system
// mount. It contains three subdirectories:
//
// - "cas": raw access to the Content Addressable Storage.
// - "outputs": outputs of builds performed using Bazel.
// - "scratch": a writable directory for testing.
defaultAttributesSetter := func(requested re_vfs.AttributesMask, attributes *re_vfs.Attributes) {}
namedAttributesFactory := re_vfs.NewInMemoryNamedAttributesFactory(
re_vfs.NewHandleAllocatingFileAllocator(
re_vfs.NewPoolBackedFileAllocator(
Expand All @@ -296,7 +322,9 @@ func main() {
rootHandleAllocator.New(),
map[path.Component]cd_vfs.InstanceNameLookupFunc{
path.MustNewComponent("blobs"): blobsDirectoryLookupFunc,
})),
},
),
),
path.MustNewComponent("outputs"): re_vfs.DirectoryChild{}.FromDirectory(outputsDirectory),
path.MustNewComponent("scratch"): re_vfs.DirectoryChild{}.FromDirectory(
re_vfs.NewInMemoryPrepopulatedDirectory(
Expand All @@ -320,7 +348,8 @@ func main() {
namedAttributesFactory,
),
),
}))
},
))

if err := mount.Expose(siblingsGroup, rootDirectory); err != nil {
return util.StatusWrap(err, "Failed to expose virtual file system mount")
Expand All @@ -334,12 +363,16 @@ func main() {
s,
grpcservers.NewActionCacheServer(
actionCache,
int(configuration.MaximumMessageSizeBytes)))
int(configuration.MaximumMessageSizeBytes),
),
)
remoteexecution.RegisterContentAddressableStorageServer(
s,
grpcservers.NewContentAddressableStorageServer(
bareContentAddressableStorage,
configuration.MaximumMessageSizeBytes))
configuration.MaximumMessageSizeBytes,
),
)
bytestream.RegisterByteStreamServer(
s,
grpcservers.NewByteStreamServer(
Expand All @@ -360,7 +393,9 @@ func main() {
LowApiVersion: &semver.SemVer{Major: 2, Minor: 0},
HighApiVersion: &semver.SemVer{Major: 2, Minor: 11},
}),
})))
}),
),
)
remoteexecution.RegisterExecutionServer(s, buildQueue)

bazeloutputservice.RegisterBazelOutputServiceServer(s, outputsDirectory)
Expand Down
Loading
Loading