Skip to content

Commit

Permalink
Tie the remote output service into bb_clientd
Browse files Browse the repository at this point in the history 
 This change extends the pathname structure offered by bb_clientd.
 Instead of letting the top-level directory provide immediate access to
 the CAS, we move this feature to a subdirectory named "cas". Two new
 top-level directories are added:

 - "outputs", which provides an implementation of the Bazel Remote Output
   Service. Details: bazelbuild/bazel#12823

 - "scratch", which provides a simple scratch space to test the behaviour
   of the FUSE file system without requiring Bazel to create an output
   directory for you.
  • Loading branch information
@EdSchouten
EdSchouten committed Feb 3, 2021
1 parent 5447385 commit d4e6ab7
Show file tree
Hide file tree
Showing 5 changed files with 106 additions and 18 deletions.
2 changes: 2 additions & 0 deletions BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ load("@bazel_gazelle//:def.bzl", "gazelle")
# gazelle:resolve proto go pkg/proto/configuration/blobstore/blobstore.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/blobstore
# gazelle:resolve proto pkg/proto/configuration/builder/builder.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/builder:builder_proto
# gazelle:resolve proto go pkg/proto/configuration/builder/builder.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/builder
# gazelle:resolve proto pkg/proto/configuration/filesystem/filesystem.proto @com_github_buildbarn_bb_remote_execution//pkg/proto/configuration/filesystem:filesystem_proto
# gazelle:resolve proto go pkg/proto/configuration/filesystem/filesystem.proto @com_github_buildbarn_bb_remote_execution//pkg/proto/configuration/filesystem
# gazelle:resolve proto pkg/proto/configuration/fuse/fuse.proto @com_github_buildbarn_bb_remote_execution//pkg/proto/configuration/fuse:fuse_proto
# gazelle:resolve proto go pkg/proto/configuration/fuse/fuse.proto @com_github_buildbarn_bb_remote_execution//pkg/proto/configuration/fuse
# gazelle:resolve proto pkg/proto/configuration/global/global.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/global:global_proto
Expand Down
3 changes: 3 additions & 0 deletions cmd/bb_clientd/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@ go_library(
"//pkg/proto/configuration/bb_clientd",
"@com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:execution",
"@com_github_buildbarn_bb_remote_execution//pkg/cas",
"@com_github_buildbarn_bb_remote_execution//pkg/filesystem",
"@com_github_buildbarn_bb_remote_execution//pkg/filesystem/fuse",
"@com_github_buildbarn_bb_remote_execution//pkg/proto/remoteoutputservice",
"@com_github_buildbarn_bb_storage//pkg/blobstore",
"@com_github_buildbarn_bb_storage//pkg/blobstore/configuration",
"@com_github_buildbarn_bb_storage//pkg/blobstore/grpcservers",
Expand All @@ -27,6 +29,7 @@ go_library(
"@com_github_buildbarn_bb_storage//pkg/filesystem/path",
"@com_github_buildbarn_bb_storage//pkg/global",
"@com_github_buildbarn_bb_storage//pkg/grpc",
"@com_github_buildbarn_bb_storage//pkg/random",
"@com_github_buildbarn_bb_storage//pkg/util",
"@com_github_hanwen_go_fuse_v2//fuse",
"@go_googleapis//google/bytestream:bytestream_go_proto",
Expand Down
90 changes: 76 additions & 14 deletions cmd/bb_clientd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,17 @@ import (
cd_fuse "github.com/buildbarn/bb-clientd/pkg/filesystem/fuse"
"github.com/buildbarn/bb-clientd/pkg/proto/configuration/bb_clientd"
re_cas "github.com/buildbarn/bb-remote-execution/pkg/cas"
re_filesystem "github.com/buildbarn/bb-remote-execution/pkg/filesystem"
re_fuse "github.com/buildbarn/bb-remote-execution/pkg/filesystem/fuse"
"github.com/buildbarn/bb-remote-execution/pkg/proto/remoteoutputservice"
blobstore_configuration "github.com/buildbarn/bb-storage/pkg/blobstore/configuration"
"github.com/buildbarn/bb-storage/pkg/blobstore/grpcservers"
"github.com/buildbarn/bb-storage/pkg/builder"
"github.com/buildbarn/bb-storage/pkg/digest"
"github.com/buildbarn/bb-storage/pkg/filesystem/path"
"github.com/buildbarn/bb-storage/pkg/global"
bb_grpc "github.com/buildbarn/bb-storage/pkg/grpc"
"github.com/buildbarn/bb-storage/pkg/random"
"github.com/buildbarn/bb-storage/pkg/util"
"github.com/hanwen/go-fuse/v2/fuse"

Expand Down Expand Up @@ -57,6 +60,12 @@ func main() {
log.Fatal(err)
}

// Storage of files created through the FUSE file system.
filePool, err := re_filesystem.NewFilePoolFromConfiguration(configuration.FilePool)
if err != nil {
log.Fatal("Failed to create file pool: ", err)
}

// Factories for FUSE nodes corresponding to plain files,
// executable files, directories and trees.
//
Expand All @@ -65,6 +74,9 @@ func main() {
// directory and tree objects. Let's not address this for the
// time being, as we mainly care about accessing individual
// files.
indexedTreeFetcher := cd_cas.NewBlobAccessIndexedTreeFetcher(
contentAddressableStorage,
int(configuration.MaximumMessageSizeBytes))
globalFileContext := NewGlobalFileContext(
context.Background(),
contentAddressableStorage,
Expand All @@ -74,11 +86,7 @@ func main() {
re_cas.NewBlobAccessDirectoryFetcher(
contentAddressableStorage,
int(configuration.MaximumMessageSizeBytes)))
globalTreeContext := NewGlobalTreeContext(
globalFileContext,
cd_cas.NewBlobAccessIndexedTreeFetcher(
contentAddressableStorage,
int(configuration.MaximumMessageSizeBytes)))
globalTreeContext := NewGlobalTreeContext(globalFileContext, indexedTreeFetcher)

// Factory function for per instance name "blobs" directories
// that give access to arbitrary files, directories and trees.
Expand Down Expand Up @@ -133,21 +141,73 @@ func main() {
return d, nil
}

// Top-level directory that parses instance names until a
// "blobs" pathname component is accessed.
rootDirectoryInodeNumberTree := re_fuse.NewRandomInodeNumberTree()
rootDirectory := cd_fuse.NewInstanceNameParsingDirectory(
rootDirectoryInodeNumberTree,
map[path.Component]cd_fuse.InstanceNameLookupFunc{
path.MustNewComponent("blobs"): blobsDirectoryLookupFunc,
// Implementation of the Remote Output Service. The Remote
// Output Service allows Bazel to place its bazel-out/
// directories on a FUSE file system, thereby allowing data to
// be loaded lazily.
var serverCallbacks re_fuse.SimpleRawFileSystemServerCallbacks
outputsInodeNumber := random.FastThreadSafeGenerator.Uint64()
outputsDirectory := cd_fuse.NewRemoteOutputServiceDirectory(
outputsInodeNumber,
random.NewFastSingleThreadedGenerator(),
serverCallbacks.EntryNotify,
func(errorLogger util.ErrorLogger, inodeNumber uint64) re_fuse.PrepopulatedDirectory {
return re_fuse.NewInMemoryPrepopulatedDirectory(
re_fuse.NewPoolBackedFileAllocator(
filePool,
errorLogger,
random.FastThreadSafeGenerator),
errorLogger,
inodeNumber,
random.FastThreadSafeGenerator,
serverCallbacks.EntryNotify)
},
contentAddressableStorage,
indexedTreeFetcher)

// Construct the top-level directory of the FUSE mount. It contains
// three subdirectories:
//
// - "cas": raw access to the Content Addressable Storage.
// - "outputs": outputs of builds performed using Bazel.
// - "scratch": a writable directory for testing.
rootInodeNumber := random.FastThreadSafeGenerator.Uint64()
casInodeNumberTree := re_fuse.NewRandomInodeNumberTree()
scratchInodeNumber := random.FastThreadSafeGenerator.Uint64()
rootDirectory := cd_fuse.NewStaticDirectory(
rootInodeNumber,
map[path.Component]cd_fuse.StaticDirectoryEntry{
path.MustNewComponent("cas"): {
Child: cd_fuse.NewInstanceNameParsingDirectory(
casInodeNumberTree,
map[path.Component]cd_fuse.InstanceNameLookupFunc{
path.MustNewComponent("blobs"): blobsDirectoryLookupFunc,
}),
InodeNumber: casInodeNumberTree.Get(),
},
path.MustNewComponent("outputs"): {
Child: outputsDirectory,
InodeNumber: outputsInodeNumber,
},
path.MustNewComponent("scratch"): {
Child: re_fuse.NewInMemoryPrepopulatedDirectory(
re_fuse.NewPoolBackedFileAllocator(
filePool,
util.DefaultErrorLogger,
random.FastThreadSafeGenerator),
util.DefaultErrorLogger,
scratchInodeNumber,
random.FastThreadSafeGenerator,
serverCallbacks.EntryNotify),
InodeNumber: scratchInodeNumber,
},
})

// Expose the FUSE file system.
var serverCallbacks re_fuse.SimpleRawFileSystemServerCallbacks
if err := re_fuse.NewMountFromConfiguration(
configuration.Fuse,
rootDirectory,
rootDirectoryInodeNumberTree.Get(),
rootInodeNumber,
&serverCallbacks,
"bb_clientd"); err != nil {
log.Fatal("Failed to mount FUSE file system: ", err)
Expand Down Expand Up @@ -177,6 +237,8 @@ func main() {
1<<16))
remoteexecution.RegisterCapabilitiesServer(s, buildQueue)
remoteexecution.RegisterExecutionServer(s, buildQueue)

remoteoutputservice.RegisterRemoteOutputServiceServer(s, outputsDirectory)
}))
}()

Expand Down
2 changes: 2 additions & 0 deletions pkg/proto/configuration/bb_clientd/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ proto_library(
srcs = ["bb_clientd.proto"],
visibility = ["//visibility:public"],
deps = [
"@com_github_buildbarn_bb_remote_execution//pkg/proto/configuration/filesystem:filesystem_proto",
"@com_github_buildbarn_bb_remote_execution//pkg/proto/configuration/fuse:fuse_proto",
"@com_github_buildbarn_bb_storage//pkg/proto/configuration/blobstore:blobstore_proto",
"@com_github_buildbarn_bb_storage//pkg/proto/configuration/builder:builder_proto",
Expand All @@ -21,6 +22,7 @@ go_proto_library(
proto = ":bb_clientd_proto",
visibility = ["//visibility:public"],
deps = [
"@com_github_buildbarn_bb_remote_execution//pkg/proto/configuration/filesystem",
"@com_github_buildbarn_bb_remote_execution//pkg/proto/configuration/fuse",
"@com_github_buildbarn_bb_storage//pkg/proto/configuration/blobstore",
"@com_github_buildbarn_bb_storage//pkg/proto/configuration/builder",
Expand Down
27 changes: 23 additions & 4 deletions pkg/proto/configuration/bb_clientd/bb_clientd.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ package buildbarn.configuration.bb_clientd;

import "pkg/proto/configuration/blobstore/blobstore.proto";
import "pkg/proto/configuration/builder/builder.proto";
import "pkg/proto/configuration/filesystem/filesystem.proto";
import "pkg/proto/configuration/fuse/fuse.proto";
import "pkg/proto/configuration/global/global.proto";
import "pkg/proto/configuration/grpc/grpc.proto";
Expand All @@ -25,15 +26,28 @@ message ApplicationConfiguration {
// Storage (CAS) are exposed for reading. The following pathname
// schemes are supported:
//
// - ${instance_name}/blobs/directory/${hash}-${size_bytes}:
// - cas/${instance_name}/blobs/directory/${hash}-${size_bytes}/:
// View the contents of a Directory object.
// - ${instance_name}/blobs/executable/${hash}-${size_bytes}:
// - cas/${instance_name}/blobs/executable/${hash}-${size_bytes}:
// Access a single file, having the executable bit (+x) set.
// - ${instance_name}/blobs/file/${hash}-${size_bytes}:
// - cas/${instance_name}/blobs/file/${hash}-${size_bytes}:
// Access a single file, having the executable bit (+x) clear.
// - ${instance_name}/blobs/tree/${hash}-${size_bytes}:
// - cas/${instance_name}/blobs/tree/${hash}-${size_bytes}/:
// View the contents of a Tree object.
//
// - outputs/${output_base}/:
// Location where Bazel may store the per-workspace bazel-out/
// directory. Files yielded by remote build actions are loaded
// lazily. Using this feature gives the same performance
// improvements as --remote_download_minimal, with the added
// advantage that remote output files remain accessible locally.
// More details: https://github.com/bazelbuild/bazel/pull/12823
//
// - scratch/:
// A writable directory where arbitrary path layouts may be
// constructed. Files that reference CAS objects may be created by
// hardlinking them from the "cas" directory.
//
// Instance names containing slashes are permitted. It automatically
// causes intermediate directories to be created. "blobs" directories
// exist at every level.
Expand All @@ -46,4 +60,9 @@ message ApplicationConfiguration {
// the key corresponds to the instance name prefix.
map<string, buildbarn.configuration.builder.SchedulerConfiguration>
schedulers = 6;

// Location where files are stored that are created in the "outputs"
// and "scratch" directories. Files that are hardlinked from the "cas"
// directory don't take up any space in the file pool.
buildbarn.configuration.filesystem.FilePoolConfiguration file_pool = 7;
}

0 comments on commit d4e6ab7

Please sign in to comment.