From e009355bb8027f5128a9d6c16927278d2de81240 Mon Sep 17 00:00:00 2001 From: William Coe Date: Thu, 2 Jul 2026 21:57:28 +0000 Subject: [PATCH] [DEV-9353] Support $optionalRef syntax JsonFileSourceProcessor Add support for $optionalRef, inspired by JSONSchema ref. It is $optionalRef and not $ref as the JSONSchema spec requires that the referenced file exist or the parser should fail, so invented this version to keep the semantics clear. We also only support file:// refs. The basic idea is that you can add a reference like so: { "metricsDatabase": { "$optionalRef": "file:///home/coe/worktmp/test.json" } } And provided the destination path exists, the entire entry (including $optionalRef key) will be replaced with what is in the destination. In the event that the destination is absent, the entire containing object is removed (ie, no metricsDatabase key in structure anymore). The idea here is that we can optionally mount things in Kubernets, such as DB secrets, config etc, and then reference it using this feature. In the event it exists, the application is configured to use the config. If it is absent, the config is removed. The result is a type of dynamic configuration based on what has been injected into the environment. Signed-off-by: William Coe --- .../zconf/sources/EnvBlobSourceProcessor.kt | 1 - .../zconf/sources/JsonFileSourceProcessor.kt | 87 ++++++- .../sources/JsonFileSourceProcessorTest.kt | 229 ++++++++++++++++-- .../optional-ref/array-ref-target.json | 3 + .../fixtures/optional-ref/invalid-target.json | 1 + .../fixtures/optional-ref/ref-leaf.json | 3 + .../fixtures/optional-ref/ref-target.json | 4 + 7 files changed, 303 insertions(+), 25 deletions(-) create mode 100644 src/nativeTest/resources/fixtures/optional-ref/array-ref-target.json create mode 100644 src/nativeTest/resources/fixtures/optional-ref/invalid-target.json create mode 100644 src/nativeTest/resources/fixtures/optional-ref/ref-leaf.json create mode 100644 src/nativeTest/resources/fixtures/optional-ref/ref-target.json diff --git a/src/nativeMain/kotlin/com/zepben/zconf/sources/EnvBlobSourceProcessor.kt b/src/nativeMain/kotlin/com/zepben/zconf/sources/EnvBlobSourceProcessor.kt index 929b87c..fc8bab8 100644 --- a/src/nativeMain/kotlin/com/zepben/zconf/sources/EnvBlobSourceProcessor.kt +++ b/src/nativeMain/kotlin/com/zepben/zconf/sources/EnvBlobSourceProcessor.kt @@ -29,7 +29,6 @@ open class EnvBlobSourceProcessor @OptIn(ExperimentalForeignApi::class) construc @OptIn(ExperimentalEncodingApi::class) override fun execute(): ConfigElement { val envValue = envFetcher(input) ?: return ConfigObject() - val json = try { val decodedValue = Base64.Default.decode(envValue) Json.Default.parseToJsonElement(postProcessEnv(decodedValue)) diff --git a/src/nativeMain/kotlin/com/zepben/zconf/sources/JsonFileSourceProcessor.kt b/src/nativeMain/kotlin/com/zepben/zconf/sources/JsonFileSourceProcessor.kt index 537d7c2..991ea4d 100644 --- a/src/nativeMain/kotlin/com/zepben/zconf/sources/JsonFileSourceProcessor.kt +++ b/src/nativeMain/kotlin/com/zepben/zconf/sources/JsonFileSourceProcessor.kt @@ -8,6 +8,8 @@ package com.zepben.zconf.sources +import kotlinx.io.files.SystemFileSystem + import com.zepben.zconf.model.CompositeConfig import com.zepben.zconf.model.ConfigArray import com.zepben.zconf.model.ConfigElement @@ -19,6 +21,8 @@ import kotlinx.io.files.SystemFileSystem import kotlinx.io.readString import kotlinx.serialization.json.* +const val OPTIONAL_REF = "\$optionalRef" + open class JsonFileSourceProcessor(input: String) : SourceProcessor(input) { private val logger = KotlinLogging.logger {} @@ -27,43 +31,108 @@ open class JsonFileSourceProcessor(input: String) : SourceProcessor(input) { try { val contents = SystemFileSystem.source(Path(input)).buffered().readString() val json = Json.Default.decodeFromString(contents) + val resolvedJson = resolveFileRefs(json, baseFile = Path(input)) + ?: throw Exception("Failed to resolve root level $OPTIONAL_REF: $json") + + return convertToIntermediateForm(resolvedJson) - return convertToIntermediateForm(json) } catch (e: Exception) { logger.error(e) { "Failed to read JSON file at $input.. skipping.." } } - return ConfigObject() } protected fun convertToIntermediateForm(json: JsonElement): ConfigObject { require(json is JsonObject) - val accumulator = ConfigObject() // just assume that we always return a full json document convertToIntermediateForm(json, "", accumulator) return accumulator } + private fun resolveFileRefs(element: JsonElement, baseFile: Path): JsonElement? { + return when (element) { + is JsonObject -> { + /* + Handle $optionalRefs. The semantics work as follows. Given the following: + "metricsDatabase": { + "$optionalRef": "file:///path/to/metrics/config.json" + } + + In the case that the config.json exists and is the valid JSON, "metricsDatabase" will refernce it. + In the case that it does not exist, "metricsDatabase" key will be removed. + In the case that it exists but is not valid JSON, then we log an error and fail. + */ + + val ref = element[OPTIONAL_REF]?.jsonPrimitive?.contentOrNull + if (ref != null) { + require(ref.startsWith("file://")) { + "$OPTIONAL_REF must use file:// scheme at $baseFile, got: $ref" + } + + val refPath = Path(ref.removePrefix("file://")) + + if (SystemFileSystem.metadataOrNull(refPath)?.isRegularFile != true) { + logger.debug { "Skipping $OPTIONAL_REF to missing file: $refPath" } + return null + } + + val contents = SystemFileSystem.source(refPath).buffered().readString() + val resolved = try { + Json.Default.parseToJsonElement(contents) + } catch (e: Exception) { + throw IllegalArgumentException( + "Invalid JSON in $OPTIONAL_REF target '$refPath' (referenced from $baseFile)", + e, + ) + } + return resolveFileRefs(resolved, refPath) + } + + JsonObject( + element.mapNotNull { (k, v) -> + resolveFileRefs(v, baseFile)?.let { k to it } + }.toMap() + ) + } + + is JsonArray -> JsonArray( + element.mapNotNull { resolveFileRefs(it, baseFile) } + ) + + else -> element + } + } + private fun convertToIntermediateForm(json: JsonElement, path: String, thing: CompositeConfig) { when (json) { is JsonNull -> return // We don't care about nulls is JsonPrimitive -> thing[path.removePrefix(".")] = json.toKotlinValue() - is JsonArray ->{ - val arr = ConfigArray().apply { thing[path] = this } // Create the ConfigArray here and don't rely on the model doing it. + is JsonArray -> { + val arr = ConfigArray().apply { + thing[path] = this + } // Create the ConfigArray here and don't rely on the model doing it. json.forEachIndexed { index, nextElement -> - val (newPath, obj) = if(nextElement is JsonObject) - "" to ConfigObject().apply { arr[index.toString()] = this } // If the nextElement is a JsonObject, we create a new ConfigObject and assign it to the array at the index. + val (newPath, obj) = if (nextElement is JsonObject) + "" to ConfigObject().apply { + arr[index.toString()] = this + } // If the nextElement is a JsonObject, we create a new ConfigObject and assign it to the array at the index. else index.toString() to arr convertToIntermediateForm(nextElement, newPath, obj) } } + is JsonObject -> json.entries.forEach { (key, nextElement) -> convertToIntermediateForm( nextElement, - key.replace(".", "__"), // NOTE: We do this key replacement of 'dots' with double underscores to be able to differentiate between nested objects and JSON object keys that contain 'dots' in them. - if(nextElement is JsonObject) ConfigObject().apply { thing[key] = this } else thing // Create the ConfigObject here and don't rely on the model doing it. + key.replace( + ".", + "__" + ), // NOTE: We do this key replacement of 'dots' with double underscores to be able to differentiate between nested objects and JSON object keys that contain 'dots' in them. + if (nextElement is JsonObject) ConfigObject().apply { + thing[key] = this + } else thing // Create the ConfigObject here and don't rely on the model doing it. ) } } diff --git a/src/nativeTest/kotlin/com/zepben/zconf/sources/JsonFileSourceProcessorTest.kt b/src/nativeTest/kotlin/com/zepben/zconf/sources/JsonFileSourceProcessorTest.kt index d2dca5a..acad27d 100644 --- a/src/nativeTest/kotlin/com/zepben/zconf/sources/JsonFileSourceProcessorTest.kt +++ b/src/nativeTest/kotlin/com/zepben/zconf/sources/JsonFileSourceProcessorTest.kt @@ -13,6 +13,9 @@ import com.zepben.zconf.model.ConfigValue import io.kotest.core.spec.style.FunSpec import io.kotest.matchers.shouldBe import kotlinx.cinterop.* +import kotlinx.io.Buffer +import kotlinx.io.files.Path +import kotlinx.io.files.SystemFileSystem import platform.posix.getcwd @OptIn(ExperimentalForeignApi::class) @@ -24,22 +27,218 @@ class JsonFileSourceProcessorTest : FunSpec({ tmp.toKString() } - test("handles non existent file") { - JsonFileSourceProcessor("$currentWorkingDir/fake/directory/fake.json").properties + val fixtures = "$currentWorkingDir/src/nativeTest/resources/fixtures" + val optionalRefFixtures = "$fixtures/optional-ref" + val testConfigDir = "$currentWorkingDir/build/zconf-test-fixtures" + + fun fileRef(absolutePath: String) = "file://$absolutePath" + + fun writeTestConfig(name: String, json: String): String { + val path = Path("$testConfigDir/$name") + path.parent?.let { SystemFileSystem.createDirectories(it) } + SystemFileSystem.sink(path).use { sink -> + val buffer = Buffer().apply { write(json.encodeToByteArray()) } + sink.write(buffer, buffer.size) + } + return path.toString() + } + + fun loadConfig(mainPath: String): ConfigObject = + JsonFileSourceProcessor(mainPath).properties as ConfigObject + + fun ConfigObject.shouldBeEmpty() { + contents().isEmpty() shouldBe true + } + + fun ConfigObject.shouldHavePaths(expected: Map?>) { + expected.forEach { (path, value) -> this[path] shouldBe value } } - test("parses a complex JSON var") { - val config = JsonFileSourceProcessor("$currentWorkingDir/src/nativeTest/resources/fixtures/sample.json").properties as ConfigObject - - config["menu.id"] shouldBe ConfigValue("file") - config["menu.value"] shouldBe ConfigValue("File") - config["menu.popup.menuitem.0.value"] shouldBe ConfigValue("New") - config["menu.popup.v1__kubernetes__zepben__com/node-class"] shouldBe ConfigValue("high-memory") - config["menu.numberThree"] shouldBe ConfigValue(3L) - config["menu.numberThreeString"] shouldBe ConfigValue("3") - config["menu.pi"] shouldBe ConfigValue(3.14) - config["menu.piString"] shouldBe ConfigValue("3.14") - config["menu.isTrue"] shouldBe ConfigValue(true) - config["menu.isTrueString"] shouldBe ConfigValue("true") + context("basic file loading") { + test("handles non existent file") { + loadConfig("$currentWorkingDir/fake/directory/fake.json").shouldBeEmpty() + } + + test("parses a complex JSON var") { + loadConfig("$fixtures/sample.json").shouldHavePaths( + mapOf( + "menu.id" to ConfigValue("file"), + "menu.value" to ConfigValue("File"), + "menu.popup.menuitem.0.value" to ConfigValue("New"), + "menu.popup.v1__kubernetes__zepben__com/node-class" to ConfigValue("high-memory"), + "menu.numberThree" to ConfigValue(3L), + "menu.numberThreeString" to ConfigValue("3"), + "menu.pi" to ConfigValue(3.14), + "menu.piString" to ConfigValue("3.14"), + "menu.isTrue" to ConfigValue(true), + "menu.isTrueString" to ConfigValue("true"), + ), + ) + } + } + + context("\$optionalRef") { + test("inlines optional ref when target exists") { + val path = writeTestConfig( + "present-ref.json", + """ + { + "app": { "name": "parent" }, + "metricsDatabase": { + "$OPTIONAL_REF": "${fileRef("$optionalRefFixtures/ref-target.json")}" + } + } + """.trimIndent(), + ) + val config = loadConfig(path) + + config.shouldHavePaths( + mapOf( + "app.name" to ConfigValue("parent"), + "metricsDatabase.host" to ConfigValue("db.example.com"), + "metricsDatabase.port" to ConfigValue(5432L), + ), + ) + } + + test("omits key when optional ref target is missing") { + val path = writeTestConfig( + "missing-ref.json", + """ + { + "app": { "name": "parent" }, + "metricsDatabase": { + "$OPTIONAL_REF": "${fileRef("$optionalRefFixtures/does-not-exist.json")}" + } + } + """.trimIndent(), + ) + val config = loadConfig(path) + + config.shouldHavePaths( + mapOf( + "app.name" to ConfigValue("parent"), + "metricsDatabase" to null, + ), + ) + } + + test("returns empty config when ref target has invalid JSON") { + val path = writeTestConfig( + "invalid-ref.json", + """ + { + "metricsDatabase": { + "$OPTIONAL_REF": "${fileRef("$optionalRefFixtures/invalid-target.json")}" + } + } + """.trimIndent(), + ) + + loadConfig(path).shouldBeEmpty() + } + + test("removes array element when optional ref target is missing") { + val path = writeTestConfig( + "array-missing-ref.json", + """ + { + "items": [ + { "$OPTIONAL_REF": "${fileRef("$optionalRefFixtures/array-ref-target.json")}" }, + { "$OPTIONAL_REF": "${fileRef("$optionalRefFixtures/missing.json")}" }, + { "static": true } + ] + } + """.trimIndent(), + ) + val config = loadConfig(path) + + config.shouldHavePaths( + mapOf( + "items.0.inlined" to ConfigValue(true), + "items.1.static" to ConfigValue(true), + "items.2.static" to null, // Actually removed missing.json ref + ), + ) + } + + test("returns empty config when optional ref uses non-file scheme") { + val path = writeTestConfig( + "bad-scheme.json", + """ + { + "metricsDatabase": { + "$OPTIONAL_REF": "http://example.com/config.json" + } + } + """.trimIndent(), + ) + + loadConfig(path).shouldBeEmpty() + } + + test("resolves root document from optional ref when target exists") { + val path = writeTestConfig( + "root-present-ref.json", + """ + { + "$OPTIONAL_REF": "${fileRef("$optionalRefFixtures/ref-target.json")}" + } + """.trimIndent(), + ) + val config = loadConfig(path) + + config.shouldHavePaths( + mapOf( + "host" to ConfigValue("db.example.com"), + "port" to ConfigValue(5432L), + ), + ) + } + + test("recursively resolves optional refs") { + val rootPath = "$testConfigDir/root-ref.json" + writeTestConfig( + "root-ref.json", + """ + { + "$OPTIONAL_REF": "${fileRef("$testConfigDir/middle-ref.json")}" + } + """.trimIndent(), + ) + writeTestConfig( + "middle-ref.json", + """ + { + "$OPTIONAL_REF": "${fileRef("$testConfigDir/leaf-ref.json")}" + } + """.trimIndent(), + ) + writeTestConfig( + "leaf-ref.json", + """ + { + "leaf-value": "resolved" + } + """.trimIndent(), + ) + val config = loadConfig(rootPath) + + config["leaf-value"] shouldBe ConfigValue("resolved") + + } + + test("returns empty config when root optional ref target is missing") { + val path = writeTestConfig( + "root-missing-ref.json", + """ + { + "$OPTIONAL_REF": "${fileRef("$optionalRefFixtures/does-not-exist.json")}" + } + """.trimIndent(), + ) + + loadConfig(path).shouldBeEmpty() + } } }) diff --git a/src/nativeTest/resources/fixtures/optional-ref/array-ref-target.json b/src/nativeTest/resources/fixtures/optional-ref/array-ref-target.json new file mode 100644 index 0000000..ea7c061 --- /dev/null +++ b/src/nativeTest/resources/fixtures/optional-ref/array-ref-target.json @@ -0,0 +1,3 @@ +{ + "inlined": true +} diff --git a/src/nativeTest/resources/fixtures/optional-ref/invalid-target.json b/src/nativeTest/resources/fixtures/optional-ref/invalid-target.json new file mode 100644 index 0000000..8cf1bc3 --- /dev/null +++ b/src/nativeTest/resources/fixtures/optional-ref/invalid-target.json @@ -0,0 +1 @@ +{ not valid json diff --git a/src/nativeTest/resources/fixtures/optional-ref/ref-leaf.json b/src/nativeTest/resources/fixtures/optional-ref/ref-leaf.json new file mode 100644 index 0000000..5d6df14 --- /dev/null +++ b/src/nativeTest/resources/fixtures/optional-ref/ref-leaf.json @@ -0,0 +1,3 @@ +{ + "deep": "value" +} diff --git a/src/nativeTest/resources/fixtures/optional-ref/ref-target.json b/src/nativeTest/resources/fixtures/optional-ref/ref-target.json new file mode 100644 index 0000000..e224547 --- /dev/null +++ b/src/nativeTest/resources/fixtures/optional-ref/ref-target.json @@ -0,0 +1,4 @@ +{ + "host": "db.example.com", + "port": 5432 +}