diff --git a/integration-tests/test-repo/files/bind/test-file.txt b/integration-tests/test-repo/files/bind/test-file.txt new file mode 100644 index 00000000..8d043451 --- /dev/null +++ b/integration-tests/test-repo/files/bind/test-file.txt @@ -0,0 +1 @@ +Test Content \ No newline at end of file diff --git a/integration-tests/test-repo/recipes/common.yml b/integration-tests/test-repo/recipes/common.yml index 59418663..fcc361af 100644 --- a/integration-tests/test-repo/recipes/common.yml +++ b/integration-tests/test-repo/recipes/common.yml @@ -147,3 +147,56 @@ modules: snippets: - '[ "$(cat /tmp/test-secret)" == "TEST_PASS" ]' + # test tmpfs mounts and is writable + - type: script + mounts: + - type: tmpfs + destination: /tmp/test-tmpfs + snippets: + - '[ -d /tmp/test-tmpfs ]' + - 'echo "Hello, World!" > /tmp/test-tmpfs/hello.txt' + - '[ "$(cat /tmp/test-tmpfs/hello.txt)" == "Hello, World!" ]' + + # tmpfs should no longer be available in later module runs + - type: script + snippets: + - '[ ! -d /tmp/test-tmpfs ]' + + # cache mount defined at recipe should be available in all module runs with shared content + - type: script + snippets: + - '[ -d /var/cache/private ]' + - 'echo "Cache Test" > /var/cache/private/cache_test.txt' + + - type: script + snippets: + - '[ "$(cat /var/cache/private/cache_test.txt)" == "Cache Test" ]' + + # test bind mounts + - type: script + mounts: + - type: bind + source: files/bind + destination: /var/bind-test + snippets: + - '[ -d /var/bind-test ]' + - '[ "$(cat /var/bind-test/test-file.txt)" == "Test Content" ]' + # test writing to bind mount + - 'echo "New Content" > /var/bind-test/new-file.txt' + - '[ "$(cat /var/bind-test/new-file.txt)" == "New Content" ]' + - 'rm /var/bind-test/new-file.txt' + + # test readonly bind mounts + - type: script + mounts: + - type: bind + source: files/bind + destination: /var/readonly-bind-test + readonly: true + snippets: + - '[ -d /var/readonly-bind-test ]' + - '[ "$(cat /var/readonly-bind-test/test-file.txt)" == "Test Content" ]' + # test that writing to readonly bind mount fails + - 'if echo "New Content" > /var/readonly-bind-test/new-file.txt 2>/dev/null; then exit 1; else exit 0; fi' + + diff --git a/integration-tests/test-repo/recipes/recipe-arm64.yml b/integration-tests/test-repo/recipes/recipe-arm64.yml index 33b15b0c..5edd28cf 100644 --- a/integration-tests/test-repo/recipes/recipe-arm64.yml +++ b/integration-tests/test-repo/recipes/recipe-arm64.yml @@ -4,6 +4,11 @@ name: cli/test-arm64 description: This is my personal OS image. base-image: quay.io/fedora/fedora-bootc image-version: latest +mounts: + - type: cache + sharing: private + id: private-cache + destination: /var/cache/private stages: - from-file: stages.yml modules: diff --git a/integration-tests/test-repo/recipes/recipe-bluefin.yml b/integration-tests/test-repo/recipes/recipe-bluefin.yml index 7cc62c5a..68db400b 100644 --- a/integration-tests/test-repo/recipes/recipe-bluefin.yml +++ b/integration-tests/test-repo/recipes/recipe-bluefin.yml @@ -6,6 +6,11 @@ base-image: ghcr.io/ublue-os/bluefin blue-build-tag: none cosign-version: none image-version: stable +mounts: + - type: cache + sharing: private + id: private-cache + destination: /var/cache/private stages: - from-file: stages.yml modules: diff --git a/integration-tests/test-repo/recipes/recipe-build-chunked-oci.yml b/integration-tests/test-repo/recipes/recipe-build-chunked-oci.yml index a1468ab4..65222852 100644 --- a/integration-tests/test-repo/recipes/recipe-build-chunked-oci.yml +++ b/integration-tests/test-repo/recipes/recipe-build-chunked-oci.yml @@ -4,6 +4,11 @@ name: cli/test-build-chunked-oci description: This is my personal OS image. base-image: quay.io/fedora/fedora-bootc image-version: latest +mounts: + - type: cache + sharing: private + id: private-cache + destination: /var/cache/private stages: - from-file: stages.yml modules: diff --git a/integration-tests/test-repo/recipes/recipe-buildah.yml b/integration-tests/test-repo/recipes/recipe-buildah.yml index eb88cf30..a3ddb531 100644 --- a/integration-tests/test-repo/recipes/recipe-buildah.yml +++ b/integration-tests/test-repo/recipes/recipe-buildah.yml @@ -4,6 +4,11 @@ name: cli/test-buildah description: This is my personal OS image. base-image: quay.io/fedora/fedora-bootc image-version: latest +mounts: + - type: cache + sharing: private + id: private-cache + destination: /var/cache/private stages: - from-file: stages.yml modules: diff --git a/integration-tests/test-repo/recipes/recipe-docker-external.yml b/integration-tests/test-repo/recipes/recipe-docker-external.yml index 388bb296..80b63375 100644 --- a/integration-tests/test-repo/recipes/recipe-docker-external.yml +++ b/integration-tests/test-repo/recipes/recipe-docker-external.yml @@ -4,6 +4,11 @@ name: cli/test-docker-external description: This is my personal OS image. base-image: quay.io/fedora/fedora-bootc image-version: latest +mounts: + - type: cache + sharing: private + id: private-cache + destination: /var/cache/private stages: - from-file: stages.yml modules: diff --git a/integration-tests/test-repo/recipes/recipe-multiplatform-build-chunked-oci.yml b/integration-tests/test-repo/recipes/recipe-multiplatform-build-chunked-oci.yml index 10069581..cfbd45f9 100644 --- a/integration-tests/test-repo/recipes/recipe-multiplatform-build-chunked-oci.yml +++ b/integration-tests/test-repo/recipes/recipe-multiplatform-build-chunked-oci.yml @@ -7,6 +7,11 @@ image-version: latest platforms: - linux/amd64 - linux/arm64 +mounts: + - type: cache + sharing: private + id: private-cache + destination: /var/cache/private stages: - from-file: stages.yml modules: diff --git a/integration-tests/test-repo/recipes/recipe-multiplatform-buildah.yml b/integration-tests/test-repo/recipes/recipe-multiplatform-buildah.yml index b3f2cc2c..c4616392 100644 --- a/integration-tests/test-repo/recipes/recipe-multiplatform-buildah.yml +++ b/integration-tests/test-repo/recipes/recipe-multiplatform-buildah.yml @@ -7,6 +7,11 @@ image-version: latest platforms: - linux/amd64 - linux/arm64 +mounts: + - type: cache + sharing: private + id: private-cache + destination: /var/cache/private stages: - from-file: stages.yml modules: diff --git a/integration-tests/test-repo/recipes/recipe-multiplatform-docker.yml b/integration-tests/test-repo/recipes/recipe-multiplatform-docker.yml index 6e360dde..d83fbb7e 100644 --- a/integration-tests/test-repo/recipes/recipe-multiplatform-docker.yml +++ b/integration-tests/test-repo/recipes/recipe-multiplatform-docker.yml @@ -7,6 +7,11 @@ image-version: latest platforms: - linux/amd64 - linux/arm64 +mounts: + - type: cache + sharing: private + id: private-cache + destination: /var/cache/private stages: - from-file: stages.yml modules: diff --git a/integration-tests/test-repo/recipes/recipe-multiplatform-podman.yml b/integration-tests/test-repo/recipes/recipe-multiplatform-podman.yml index 896d8974..e3ccc421 100644 --- a/integration-tests/test-repo/recipes/recipe-multiplatform-podman.yml +++ b/integration-tests/test-repo/recipes/recipe-multiplatform-podman.yml @@ -7,6 +7,11 @@ image-version: latest platforms: - linux/amd64 - linux/arm64 +mounts: + - type: cache + sharing: private + id: private-cache + destination: /var/cache/private stages: - from-file: stages.yml modules: diff --git a/integration-tests/test-repo/recipes/recipe-multiplatform-rechunk.yml b/integration-tests/test-repo/recipes/recipe-multiplatform-rechunk.yml index 204ab53d..3188db94 100644 --- a/integration-tests/test-repo/recipes/recipe-multiplatform-rechunk.yml +++ b/integration-tests/test-repo/recipes/recipe-multiplatform-rechunk.yml @@ -7,6 +7,11 @@ image-version: latest platforms: - linux/amd64 - linux/arm64 +mounts: + - type: cache + sharing: private + id: private-cache + destination: /var/cache/private stages: - from-file: stages.yml modules: diff --git a/integration-tests/test-repo/recipes/recipe-podman.yml b/integration-tests/test-repo/recipes/recipe-podman.yml index 93ddb707..8c1fba46 100644 --- a/integration-tests/test-repo/recipes/recipe-podman.yml +++ b/integration-tests/test-repo/recipes/recipe-podman.yml @@ -4,6 +4,11 @@ name: cli/test-podman description: This is my personal OS image. base-image: quay.io/fedora/fedora-bootc image-version: latest +mounts: + - type: cache + sharing: private + id: private-cache + destination: /var/cache/private stages: - from-file: stages.yml modules: diff --git a/integration-tests/test-repo/recipes/recipe-rechunk.yml b/integration-tests/test-repo/recipes/recipe-rechunk.yml index 699320f1..9bb6c0a3 100644 --- a/integration-tests/test-repo/recipes/recipe-rechunk.yml +++ b/integration-tests/test-repo/recipes/recipe-rechunk.yml @@ -4,6 +4,11 @@ name: cli/test-rechunk description: This is my personal OS image. base-image: quay.io/fedora/fedora-bootc image-version: latest +mounts: + - type: cache + sharing: private + id: private-cache + destination: /var/cache/private stages: - from-file: stages.yml modules: diff --git a/integration-tests/test-repo/recipes/recipe.yml b/integration-tests/test-repo/recipes/recipe.yml index 901e9d84..63c49a16 100644 --- a/integration-tests/test-repo/recipes/recipe.yml +++ b/integration-tests/test-repo/recipes/recipe.yml @@ -6,6 +6,11 @@ base-image: quay.io/fedora/fedora-bootc blue-build-tag: none cosign-version: none image-version: latest +mounts: + - type: cache + sharing: private + id: private-cache + destination: /var/cache/private stages: - from-file: stages.yml modules: diff --git a/recipe/src/lib.rs b/recipe/src/lib.rs index 93274c54..9857f68d 100644 --- a/recipe/src/lib.rs +++ b/recipe/src/lib.rs @@ -2,6 +2,7 @@ mod akmods_info; mod maybe_version; mod module; mod module_ext; +mod mount; mod recipe; mod stage; mod stages_ext; @@ -15,6 +16,7 @@ pub use akmods_info::*; pub use maybe_version::*; pub use module::*; pub use module_ext::*; +pub use mount::*; pub use recipe::*; pub use stage::*; pub use stages_ext::*; diff --git a/recipe/src/module.rs b/recipe/src/module.rs index 2e501f78..cf896fe6 100644 --- a/recipe/src/module.rs +++ b/recipe/src/module.rs @@ -11,7 +11,7 @@ use miette::{Result, bail}; use serde::{Deserialize, Serialize}; use serde_yaml::Value; -use crate::{AkmodsInfo, ModuleExt, base_recipe_path}; +use crate::{AkmodsInfo, ModuleExt, base_recipe_path, mount::Mount}; mod type_ver; @@ -39,6 +39,10 @@ pub struct ModuleRequiredFields { #[serde(skip_serializing_if = "Vec::is_empty", default)] pub secrets: Vec, + #[builder(default)] + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub mounts: Vec, + #[serde(flatten)] #[builder(default, into)] pub config: IndexMap, diff --git a/recipe/src/mount.rs b/recipe/src/mount.rs new file mode 100644 index 00000000..8ce7b686 --- /dev/null +++ b/recipe/src/mount.rs @@ -0,0 +1,112 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub enum MountCacheSharing { + /// The cache is shared between all builds. + #[serde(rename = "shared")] + Shared, + + /// The cache is private to the current build. + #[serde(rename = "private")] + Private, + + /// The cache is shared between builds, but only one build can use it at a time. + #[serde(rename = "locked")] + Locked, +} +impl std::fmt::Display for MountCacheSharing { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Shared => write!(f, "shared"), + Self::Private => write!(f, "private"), + Self::Locked => write!(f, "locked"), + } + } +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(tag = "type")] +pub enum Mount { + /// A bind mount, which mounts a file or directory from the host. + #[serde(rename = "bind")] + Bind { + /// The source path on the host. + source: String, + /// The destination path in the container. + destination: String, + /// Whether the mount is read-only. + #[serde(default, rename = "readonly")] + readonly: bool, + }, + + /// A tmpfs mount, which mounts a temporary file system in memory. + #[serde(rename = "tmpfs")] + Tmpfs { + /// The destination path. + destination: String, + /// The size of the tmpfs. Can be specified in bytes or with a suffix (e.g. "100m" for 100 megabytes). + #[serde(skip_serializing_if = "Option::is_none")] + size: Option, + }, + + /// A cache mount, which mounts a cache directory that can be shared between builds. + #[serde(rename = "cache")] + Cache { + /// The destination path. + destination: String, + /// The cache ID, which is used to identify the cache. + #[serde(skip_serializing_if = "Option::is_none")] + id: Option, + /// The cache sharing mode. + #[serde(skip_serializing_if = "Option::is_none")] + sharing: Option, + }, +} + +impl std::fmt::Display for Mount { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Bind { + source, + destination, + readonly, + } => { + write!(f, "type=bind,source={source},dst={destination}")?; + if *readonly { + write!(f, ",readonly")?; + } + } + Self::Tmpfs { destination, size } => { + write!(f, "type=tmpfs,dst={destination}")?; + if let Some(size) = size { + write!(f, ",size={size}")?; + } + } + Self::Cache { + destination, + id, + sharing, + } => { + write!(f, "type=cache")?; + if let Some(sharing) = sharing { + write!(f, ",sharing={sharing}")?; + } + write!(f, ",dst={destination}")?; + if let Some(id) = id { + write!(f, ",id={id}")?; + } + } + } + Ok(()) + } +} + +impl Mount { + #[must_use] + pub const fn oci_suffix(&self) -> &'static str { + match self { + Self::Bind { .. } => ",z", + _ => "", + } + } +} diff --git a/recipe/src/recipe.rs b/recipe/src/recipe.rs index f32a01f7..f7f4657c 100644 --- a/recipe/src/recipe.rs +++ b/recipe/src/recipe.rs @@ -15,7 +15,7 @@ use miette::{Context, IntoDiagnostic, Result}; use oci_client::Reference; use serde::{Deserialize, Serialize}; -use crate::{Module, ModuleExt, StagesExt, maybe_version::MaybeVersion}; +use crate::{Module, ModuleExt, StagesExt, maybe_version::MaybeVersion, mount::Mount}; /// The build recipe. /// @@ -90,6 +90,11 @@ pub struct Recipe { /// This hashmap provides custom labels from ther use to the image #[serde(skip_serializing_if = "Option::is_none")] pub labels: Option>, + + /// The mounts to add to the image. + #[serde(skip_serializing_if = "Vec::is_empty", default)] + #[builder(default)] + pub mounts: Vec, } impl Recipe { diff --git a/template/src/lib.rs b/template/src/lib.rs index 9e6bd111..8d14049b 100644 --- a/template/src/lib.rs +++ b/template/src/lib.rs @@ -1,6 +1,6 @@ -use std::{borrow::Cow, collections::BTreeMap, fs, path::Path, process}; +use std::{borrow::Cow, collections::BTreeMap, fmt::Write, fs, path::Path, process}; -use blue_build_recipe::{MaybeVersion, Recipe}; +use blue_build_recipe::{MaybeVersion, ModuleRequiredFields, Recipe}; use blue_build_utils::{ constants::{CONFIG_PATH, CONTAINER_FILE, CONTAINERFILES_PATH, COSIGN_PUB_PATH, FILES_PATH}, container::Tag, @@ -80,6 +80,23 @@ impl ContainerFileTemplate<'_> { } ) } + + fn user_mounts(&self, module: &ModuleRequiredFields) -> String { + let mut s = self.recipe.mounts.iter().chain(module.mounts.iter()).fold( + String::new(), + |mut acc, mount| { + let suffix: &str = match self.build_engine { + BuildEngine::Oci => mount.oci_suffix(), + BuildEngine::Docker => "", + }; + + writeln!(acc, "--mount={mount}{suffix} \\").unwrap(); + acc + }, + ); + s.pop(); // Avoid trailing newline + s + } } #[derive(Debug, Clone, Template, Builder)] diff --git a/template/templates/modules/modules.j2 b/template/templates/modules/modules.j2 index d6c67033..d72977f9 100644 --- a/template/templates/modules/modules.j2 +++ b/template/templates/modules/modules.j2 @@ -58,6 +58,9 @@ RUN \ {{ cache_mount }},dst=/var/cache/apt,id=apt-{{ cache_name }} \ {{ cache_mount }},dst=/var/cache/pacman,id=pacman-{{ cache_name }} \ + {#- User defined mounts #} + {{ user_mounts(&module) }} + {#- Secret environment variables #} {%- for secret_var in module.secrets.envs() %} {{ secret_var }} \ diff --git a/test-files/recipes/recipe-pass.yml b/test-files/recipes/recipe-pass.yml index 9eabb78c..447c5362 100644 --- a/test-files/recipes/recipe-pass.yml +++ b/test-files/recipes/recipe-pass.yml @@ -58,3 +58,15 @@ modules: from: fedora-test src: /test.txt dest: / + - type: test-module + source: local + mounts: + - type: tmpfs + destination: /tmp/test +mounts: + - type: cache + id: downloads-cli-test-43 + destination: /var/cache/downloads + sharing: locked + - type: tmpfs + destination: /tmp diff --git a/test-files/schema/recipe-v1.json b/test-files/schema/recipe-v1.json index 56c8117a..2232bcac 100644 --- a/test-files/schema/recipe-v1.json +++ b/test-files/schema/recipe-v1.json @@ -50,6 +50,13 @@ "$ref": "#/$defs/ModuleEntry" }, "description": "A list of [modules](https://blue-build.org/reference/module/) that is executed in order. Multiple of the same module can be included.\n\nEach item in this list should have at least a `type:` or be specified to be included from an external file in the `recipes/` directory with `from-file:`." + }, + "mounts": { + "type": "array", + "items": { + "$ref": "#/$defs/Mount" + }, + "description": "A list of mounts that will be mounted in each RUN stage." } }, "required": [ @@ -78,9 +85,118 @@ }, { "$ref": "#/$defs/ImportedModule" + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "mounts": { + "type": "array", + "items": { + "$ref": "#/$defs/Mount" + }, + "description": "A list of mounts that will be mounted for this module." + } + } + } + ] + }, + "Mount": { + "oneOf": [ + { + "$ref": "#/$defs/MountBind" + }, + { + "$ref": "#/$defs/MountTmpFs" + }, + { + "$ref": "#/$defs/MountCache" } ] }, + "MountBind": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "bind", + "description": "A bind mount from the host system." + }, + "source": { + "type": "string", + "description": "The source path on the host system to mount." + }, + "destination": { + "type": "string", + "description": "The destination path to mount to." + }, + "readonly": { + "type": "boolean", + "description": "Whether the mount should be read-only." + } + }, + "required": [ + "type", + "source", + "destination" + ] + }, + "MountTmpFs": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "tmpfs", + "description": "A temporary file system mount." + }, + "destination": { + "type": "string", + "description": "The destination path to mount to." + }, + "size": { + "type": "string", + "description": "The size of the tmpfs mount." + } + }, + "required": [ + "type", + "destination" + ] + }, + "MountCache": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "cache", + "description": "A cache mount." + }, + "destination": { + "type": "string", + "description": "The destination path to mount to." + }, + "id": { + "type": "string", + "description": "An identifier for the cache." + }, + "sharing": { + "$ref": "#/$defs/MountCacheSharing", + "description": "The sharing mode of the cache." + } + }, + "required": [ + "type", + "destination" + ] + }, + "MountCacheSharing": { + "type": "string", + "enum": [ + "shared", + "private", + "locked" + ] + }, "ImportedModule": { "type": "object", "properties": { diff --git a/utils/src/constants.rs b/utils/src/constants.rs index 13ae5867..5edc4354 100644 --- a/utils/src/constants.rs +++ b/utils/src/constants.rs @@ -125,7 +125,7 @@ pub const GITHUB_CHAR_LIMIT: usize = 8100; // Magic number accepted by Github pub const DEFAULT_MAX_LAYERS: NonZeroU32 = NonZeroU32::new(128).unwrap(); // Schema -pub const SCHEMA_BASE_URL: &str = "https://schema.blue-build.org"; +pub const SCHEMA_BASE_URL: &str = "https://mounts.schema-e29.pages.dev"; pub const RECIPE_V1_SCHEMA_URL: &str = concat!(SCHEMA_BASE_URL, "/recipe-v1.json"); pub const STAGE_V1_SCHEMA_URL: &str = concat!(SCHEMA_BASE_URL, "/stage-v1.json"); pub const MODULE_V1_SCHEMA_URL: &str = concat!(SCHEMA_BASE_URL, "/module-v1.json");