Skip to content

Add a container image for partitioning ephemeral disks #348

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
EdSchouten merged 1 commit into from
Jun 23, 2026
+518 −0
Merged

Add a container image for partitioning ephemeral disks #348

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
131 changes: 131 additions & 0 deletions .github/workflows/main.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,20 @@
"path": "bb_storage.linux_amd64"
}
},
{
"if": "matrix.host.upload",
"name": "linux_amd64: copy partition_ephemeral_disks",
"run": "rm -f partition_ephemeral_disks.linux_amd64 && bazel run --run_under cp --platforms=@com_github_buildbarn_bb_storage//tools/platforms:linux_amd64 //cmd/partition_ephemeral_disks $(pwd)/partition_ephemeral_disks.linux_amd64"
},
{
"if": "matrix.host.upload",
"name": "linux_amd64: upload partition_ephemeral_disks",
"uses": "actions/upload-artifact@v4",
"with": {
"name": "partition_ephemeral_disks.linux_amd64",
"path": "partition_ephemeral_disks.linux_amd64"
}
},
{
"if": "matrix.host.upload",
"name": "linux_amd64: copy sync_jwks_to_configmap",
Expand Down Expand Up @@ -126,6 +140,20 @@
"path": "bb_storage.linux_amd64_v3"
}
},
{
"if": "matrix.host.upload",
"name": "linux_amd64_v3: copy partition_ephemeral_disks",
"run": "rm -f partition_ephemeral_disks.linux_amd64_v3 && bazel run --run_under cp --platforms=@com_github_buildbarn_bb_storage//tools/platforms:linux_amd64_v3 //cmd/partition_ephemeral_disks $(pwd)/partition_ephemeral_disks.linux_amd64_v3"
},
{
"if": "matrix.host.upload",
"name": "linux_amd64_v3: upload partition_ephemeral_disks",
"uses": "actions/upload-artifact@v4",
"with": {
"name": "partition_ephemeral_disks.linux_amd64_v3",
"path": "partition_ephemeral_disks.linux_amd64_v3"
}
},
{
"if": "matrix.host.upload",
"name": "linux_amd64_v3: copy sync_jwks_to_configmap",
Expand Down Expand Up @@ -187,6 +215,20 @@
"path": "bb_storage.linux_386"
}
},
{
"if": "matrix.host.upload",
"name": "linux_386: copy partition_ephemeral_disks",
"run": "rm -f partition_ephemeral_disks.linux_386 && bazel run --run_under cp --platforms=@com_github_buildbarn_bb_storage//tools/platforms:linux_386 //cmd/partition_ephemeral_disks $(pwd)/partition_ephemeral_disks.linux_386"
},
{
"if": "matrix.host.upload",
"name": "linux_386: upload partition_ephemeral_disks",
"uses": "actions/upload-artifact@v4",
"with": {
"name": "partition_ephemeral_disks.linux_386",
"path": "partition_ephemeral_disks.linux_386"
}
},
{
"if": "matrix.host.upload",
"name": "linux_386: copy sync_jwks_to_configmap",
Expand Down Expand Up @@ -248,6 +290,20 @@
"path": "bb_storage.linux_arm"
}
},
{
"if": "matrix.host.upload",
"name": "linux_arm: copy partition_ephemeral_disks",
"run": "rm -f partition_ephemeral_disks.linux_arm && bazel run --run_under cp --platforms=@com_github_buildbarn_bb_storage//tools/platforms:linux_arm //cmd/partition_ephemeral_disks $(pwd)/partition_ephemeral_disks.linux_arm"
},
{
"if": "matrix.host.upload",
"name": "linux_arm: upload partition_ephemeral_disks",
"uses": "actions/upload-artifact@v4",
"with": {
"name": "partition_ephemeral_disks.linux_arm",
"path": "partition_ephemeral_disks.linux_arm"
}
},
{
"if": "matrix.host.upload",
"name": "linux_arm: copy sync_jwks_to_configmap",
Expand Down Expand Up @@ -309,6 +365,20 @@
"path": "bb_storage.linux_arm64"
}
},
{
"if": "matrix.host.upload",
"name": "linux_arm64: copy partition_ephemeral_disks",
"run": "rm -f partition_ephemeral_disks.linux_arm64 && bazel run --run_under cp --platforms=@com_github_buildbarn_bb_storage//tools/platforms:linux_arm64 //cmd/partition_ephemeral_disks $(pwd)/partition_ephemeral_disks.linux_arm64"
},
{
"if": "matrix.host.upload",
"name": "linux_arm64: upload partition_ephemeral_disks",
"uses": "actions/upload-artifact@v4",
"with": {
"name": "partition_ephemeral_disks.linux_arm64",
"path": "partition_ephemeral_disks.linux_arm64"
}
},
{
"if": "matrix.host.upload",
"name": "linux_arm64: copy sync_jwks_to_configmap",
Expand Down Expand Up @@ -370,6 +440,20 @@
"path": "bb_storage.darwin_amd64"
}
},
{
"if": "matrix.host.upload",
"name": "darwin_amd64: copy partition_ephemeral_disks",
"run": "rm -f partition_ephemeral_disks.darwin_amd64 && bazel run --run_under cp --platforms=@com_github_buildbarn_bb_storage//tools/platforms:darwin_amd64 //cmd/partition_ephemeral_disks $(pwd)/partition_ephemeral_disks.darwin_amd64"
},
{
"if": "matrix.host.upload",
"name": "darwin_amd64: upload partition_ephemeral_disks",
"uses": "actions/upload-artifact@v4",
"with": {
"name": "partition_ephemeral_disks.darwin_amd64",
"path": "partition_ephemeral_disks.darwin_amd64"
}
},
{
"if": "matrix.host.upload",
"name": "darwin_amd64: copy sync_jwks_to_configmap",
Expand Down Expand Up @@ -431,6 +515,20 @@
"path": "bb_storage.darwin_arm64"
}
},
{
"if": "matrix.host.upload",
"name": "darwin_arm64: copy partition_ephemeral_disks",
"run": "rm -f partition_ephemeral_disks.darwin_arm64 && bazel run --run_under cp --platforms=@com_github_buildbarn_bb_storage//tools/platforms:darwin_arm64 //cmd/partition_ephemeral_disks $(pwd)/partition_ephemeral_disks.darwin_arm64"
},
{
"if": "matrix.host.upload",
"name": "darwin_arm64: upload partition_ephemeral_disks",
"uses": "actions/upload-artifact@v4",
"with": {
"name": "partition_ephemeral_disks.darwin_arm64",
"path": "partition_ephemeral_disks.darwin_arm64"
}
},
{
"if": "matrix.host.upload",
"name": "darwin_arm64: copy sync_jwks_to_configmap",
Expand Down Expand Up @@ -492,6 +590,20 @@
"path": "bb_storage.freebsd_amd64"
}
},
{
"if": "matrix.host.upload",
"name": "freebsd_amd64: copy partition_ephemeral_disks",
"run": "rm -f partition_ephemeral_disks.freebsd_amd64 && bazel run --run_under cp --platforms=@com_github_buildbarn_bb_storage//tools/platforms:freebsd_amd64 //cmd/partition_ephemeral_disks $(pwd)/partition_ephemeral_disks.freebsd_amd64"
},
{
"if": "matrix.host.upload",
"name": "freebsd_amd64: upload partition_ephemeral_disks",
"uses": "actions/upload-artifact@v4",
"with": {
"name": "partition_ephemeral_disks.freebsd_amd64",
"path": "partition_ephemeral_disks.freebsd_amd64"
}
},
{
"if": "matrix.host.upload",
"name": "freebsd_amd64: copy sync_jwks_to_configmap",
Expand Down Expand Up @@ -553,6 +665,20 @@
"path": "bb_storage.windows_amd64.exe"
}
},
{
"if": "matrix.host.upload",
"name": "windows_amd64: copy partition_ephemeral_disks",
"run": "rm -f partition_ephemeral_disks.windows_amd64.exe && bazel run --run_under cp --platforms=@com_github_buildbarn_bb_storage//tools/platforms:windows_amd64 //cmd/partition_ephemeral_disks $(pwd)/partition_ephemeral_disks.windows_amd64.exe"
},
{
"if": "matrix.host.upload",
"name": "windows_amd64: upload partition_ephemeral_disks",
"uses": "actions/upload-artifact@v4",
"with": {
"name": "partition_ephemeral_disks.windows_amd64",
"path": "partition_ephemeral_disks.windows_amd64.exe"
}
},
{
"if": "matrix.host.upload",
"name": "windows_amd64: copy sync_jwks_to_configmap",
Expand Down Expand Up @@ -595,6 +721,11 @@
"name": "Push container bb_storage:bb_storage",
"run": "bazel run --stamp //cmd/bb_storage:bb_storage_container_push"
},
{
"if": "matrix.host.upload",
"name": "Push container partition_ephemeral_disks:partition_ephemeral_disks",
"run": "bazel run --stamp //cmd/partition_ephemeral_disks:partition_ephemeral_disks_container_push"
},
{
"if": "matrix.host.upload",
"name": "Push container sync_jwks_to_configmap:sync_jwks_to_configmap",
Expand Down
31 changes: 31 additions & 0 deletions cmd/partition_ephemeral_disks/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
load("@rules_go//go:def.bzl", "go_binary", "go_library")
load("//tools:container.bzl", "container_push_official", "multiarch_go_image")

go_library(
name = "partition_ephemeral_disks_lib",
srcs = ["main.go"],
importpath = "github.com/buildbarn/bb-storage/cmd/partition_ephemeral_disks",
visibility = ["//visibility:private"],
deps = [
"//pkg/proto/configuration/partition_ephemeral_disks",
"//pkg/util",
],
)

go_binary(
name = "partition_ephemeral_disks",
embed = [":partition_ephemeral_disks_lib"],
pure = "on",
visibility = ["//visibility:public"],
)

multiarch_go_image(
name = "partition_ephemeral_disks_container",
binary = ":partition_ephemeral_disks",
)

container_push_official(
name = "partition_ephemeral_disks_container_push",
component = "bb-storage",
image = ":partition_ephemeral_disks_container",
)
115 changes: 115 additions & 0 deletions cmd/partition_ephemeral_disks/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
package main

import (
"errors"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strconv"

"github.com/buildbarn/bb-storage/pkg/proto/configuration/partition_ephemeral_disks"
"github.com/buildbarn/bb-storage/pkg/util"
)

// partition_ephemeral_disks: Place all local SSDs of a cloud compute
// instance in a single LVM2 volume group, and partition it.
//
// This command can be run as part of a daemonset on all nodes in a
// given Kubernetes node group to ensure that the ephemeral storage
// present on these systems is partitioned properly. That way other pods
// can make use of raw block devices with fine-grained sizes.

func run(command string, arguments ...string) error {
cmd := exec.Command(command, arguments...)
// lvcreate may hang if the following environment variable is
// not set, as it can't post device creation notifications to
// udev on the host system. By setting this environment
// variable, lvcreate will create new device nodes under /dev
// directly.
cmd.Env = append(cmd.Environ(), "DM_DISABLE_UDEV=y")
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
return cmd.Run()
}

func main() {
if len(os.Args) != 2 {
log.Fatal("Usage: partition_ephemeral_disks partition_ephemeral_disks.jsonnet")
}
var configuration partition_ephemeral_disks.ApplicationConfiguration
if err := util.UnmarshalConfigurationFromFile(os.Args[1], &configuration); err != nil {
log.Fatalf("Failed to read configuration from %s: %s", os.Args[1], err)
}

// Determine which devices need to go in the volume group.
unexpandedDevicePaths, err := filepath.Glob(configuration.EphemeralDiskDevicesPattern)
if err != nil {
log.Fatal("Failed to obtain paths of ephemeral disk devices: ", err)
}
expandedDevicePaths := make(map[string]struct{}, len(unexpandedDevicePaths))
for _, unexpandedDevicePath := range unexpandedDevicePaths {
expandedDevicePath, err := filepath.EvalSymlinks(unexpandedDevicePath)
if err != nil {
log.Fatalf("Cannot evaluate symbolic links for path %#v: %s", unexpandedDevicePath, err)
}
expandedDevicePaths[expandedDevicePath] = struct{}{}
}
if len(expandedDevicePaths) == 0 {
log.Fatalf("Pattern %#v does not expand to any ephemeral disk devices", configuration.EphemeralDiskDevicesPattern)
}

// Create the volume group if needed.
volumeGroupName := "ephemeral"
if err := run("vgs", volumeGroupName); err != nil {
var exitErr *exec.ExitError
if !errors.As(err, &exitErr) || exitErr.ExitCode() != 5 {
log.Fatalf("Failed to determine existence of volume group %#v: %s", volumeGroupName, err)
}

arguments := []string{"--force", volumeGroupName}
for expandedDevicePath := range expandedDevicePaths {
arguments = append(arguments, expandedDevicePath)
}
if err := run("vgcreate", arguments...); err != nil {
log.Fatalf("Failed to create volume group %#v: %s", volumeGroupName, err)
}
}

// Check validity of the provided partitions.
totalPercentage := int32(0)
for _, partition := range configuration.Partitions {
if partition.SizePercentage < 1 || partition.SizePercentage > 100 {
log.Fatalf("Partition %#v has illegal size %d", partition.Name, partition.SizePercentage)
}
totalPercentage += partition.SizePercentage
}
if totalPercentage != 100 {
log.Fatal("Partition sizes do not add up to 100%")
}

// Create the partitions inside the ephemeral volume group.
var baseArguments []string
if len(expandedDevicePaths) > 1 {
baseArguments = []string{
"--type", "raid0",
"--stripes", strconv.FormatInt(int64(len(expandedDevicePaths)), 10),
"--stripesize", "2048",
}
}
for _, partition := range configuration.Partitions {
if err := run("lvcreate", append(
append([]string(nil), baseArguments...),
"--extents", fmt.Sprintf("%d%%VG", partition.SizePercentage),
"--name", partition.Name,
"--wipesignatures", "n",
volumeGroupName,
)...); err != nil {
var exitErr *exec.ExitError
if !errors.As(err, &exitErr) || exitErr.ExitCode() != 5 {
log.Fatalf("Failed to create logical volume %#v: %s", partition.Name, err)
}
}
}
}
24 changes: 24 additions & 0 deletions pkg/proto/configuration/partition_ephemeral_disks/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
load("@protobuf//bazel:proto_library.bzl", "proto_library")
load("@rules_go//go:def.bzl", "go_library")
load("@rules_go//proto:def.bzl", "go_proto_library")

proto_library(
name = "partition_ephemeral_disks_proto",
srcs = ["partition_ephemeral_disks.proto"],
import_prefix = "github.com/buildbarn/bb-storage",
visibility = ["//visibility:public"],
)

go_proto_library(
name = "partition_ephemeral_disks_go_proto",
importpath = "github.com/buildbarn/bb-storage/pkg/proto/configuration/partition_ephemeral_disks",
proto = ":partition_ephemeral_disks_proto",
visibility = ["//visibility:public"],
)

go_library(
name = "partition_ephemeral_disks",
embed = [":partition_ephemeral_disks_go_proto"],
importpath = "github.com/buildbarn/bb-storage/pkg/proto/configuration/partition_ephemeral_disks",
visibility = ["//visibility:public"],
)
Loading