diff --git a/src/nativeMain/kotlin/com/zepben/zconf/sources/EnvBlobSourceProcessor.kt b/src/nativeMain/kotlin/com/zepben/zconf/sources/EnvBlobSourceProcessor.kt index 929b87c..fc8bab8 100644 --- a/src/nativeMain/kotlin/com/zepben/zconf/sources/EnvBlobSourceProcessor.kt +++ b/src/nativeMain/kotlin/com/zepben/zconf/sources/EnvBlobSourceProcessor.kt @@ -29,7 +29,6 @@ open class EnvBlobSourceProcessor @OptIn(ExperimentalForeignApi::class) construc @OptIn(ExperimentalEncodingApi::class) override fun execute(): ConfigElement { val envValue = envFetcher(input) ?: return ConfigObject() - val json = try { val decodedValue = Base64.Default.decode(envValue) Json.Default.parseToJsonElement(postProcessEnv(decodedValue)) diff --git a/src/nativeMain/kotlin/com/zepben/zconf/sources/JsonFileSourceProcessor.kt b/src/nativeMain/kotlin/com/zepben/zconf/sources/JsonFileSourceProcessor.kt index 537d7c2..991ea4d 100644 --- a/src/nativeMain/kotlin/com/zepben/zconf/sources/JsonFileSourceProcessor.kt +++ b/src/nativeMain/kotlin/com/zepben/zconf/sources/JsonFileSourceProcessor.kt @@ -8,6 +8,8 @@ package com.zepben.zconf.sources +import kotlinx.io.files.SystemFileSystem + import com.zepben.zconf.model.CompositeConfig import com.zepben.zconf.model.ConfigArray import com.zepben.zconf.model.ConfigElement @@ -19,6 +21,8 @@ import kotlinx.io.files.SystemFileSystem import kotlinx.io.readString import kotlinx.serialization.json.* +const val OPTIONAL_REF = "\$optionalRef" + open class JsonFileSourceProcessor(input: String) : SourceProcessor(input) { private val logger = KotlinLogging.logger {} @@ -27,43 +31,108 @@ open class JsonFileSourceProcessor(input: String) : SourceProcessor(input) { try { val contents = SystemFileSystem.source(Path(input)).buffered().readString() val json = Json.Default.decodeFromString(contents) + val resolvedJson = resolveFileRefs(json, baseFile = Path(input)) + ?: throw Exception("Failed to resolve root level $OPTIONAL_REF: $json") + + return convertToIntermediateForm(resolvedJson) - return convertToIntermediateForm(json) } catch (e: Exception) { logger.error(e) { "Failed to read JSON file at $input.. skipping.." } } - return ConfigObject() } protected fun convertToIntermediateForm(json: JsonElement): ConfigObject { require(json is JsonObject) - val accumulator = ConfigObject() // just assume that we always return a full json document convertToIntermediateForm(json, "", accumulator) return accumulator } + private fun resolveFileRefs(element: JsonElement, baseFile: Path): JsonElement? { + return when (element) { + is JsonObject -> { + /* + Handle $optionalRefs. The semantics work as follows. Given the following: + "metricsDatabase": { + "$optionalRef": "file:///path/to/metrics/config.json" + } + + In the case that the config.json exists and is the valid JSON, "metricsDatabase" will refernce it. + In the case that it does not exist, "metricsDatabase" key will be removed. + In the case that it exists but is not valid JSON, then we log an error and fail. + */ + + val ref = element[OPTIONAL_REF]?.jsonPrimitive?.contentOrNull + if (ref != null) { + require(ref.startsWith("file://")) { + "$OPTIONAL_REF must use file:// scheme at $baseFile, got: $ref" + } + + val refPath = Path(ref.removePrefix("file://")) + + if (SystemFileSystem.metadataOrNull(refPath)?.isRegularFile != true) { + logger.debug { "Skipping $OPTIONAL_REF to missing file: $refPath" } + return null + } + + val contents = SystemFileSystem.source(refPath).buffered().readString() + val resolved = try { + Json.Default.parseToJsonElement(contents) + } catch (e: Exception) { + throw IllegalArgumentException( + "Invalid JSON in $OPTIONAL_REF target '$refPath' (referenced from $baseFile)", + e, + ) + } + return resolveFileRefs(resolved, refPath) + } + + JsonObject( + element.mapNotNull { (k, v) -> + resolveFileRefs(v, baseFile)?.let { k to it } + }.toMap() + ) + } + + is JsonArray -> JsonArray( + element.mapNotNull { resolveFileRefs(it, baseFile) } + ) + + else -> element + } + } + private fun convertToIntermediateForm(json: JsonElement, path: String, thing: CompositeConfig) { when (json) { is JsonNull -> return // We don't care about nulls is JsonPrimitive -> thing[path.removePrefix(".")] = json.toKotlinValue() - is JsonArray ->{ - val arr = ConfigArray().apply { thing[path] = this } // Create the ConfigArray here and don't rely on the model doing it. + is JsonArray -> { + val arr = ConfigArray().apply { + thing[path] = this + } // Create the ConfigArray here and don't rely on the model doing it. json.forEachIndexed { index, nextElement -> - val (newPath, obj) = if(nextElement is JsonObject) - "" to ConfigObject().apply { arr[index.toString()] = this } // If the nextElement is a JsonObject, we create a new ConfigObject and assign it to the array at the index. + val (newPath, obj) = if (nextElement is JsonObject) + "" to ConfigObject().apply { + arr[index.toString()] = this + } // If the nextElement is a JsonObject, we create a new ConfigObject and assign it to the array at the index. else index.toString() to arr convertToIntermediateForm(nextElement, newPath, obj) } } + is JsonObject -> json.entries.forEach { (key, nextElement) -> convertToIntermediateForm( nextElement, - key.replace(".", "__"), // NOTE: We do this key replacement of 'dots' with double underscores to be able to differentiate between nested objects and JSON object keys that contain 'dots' in them. - if(nextElement is JsonObject) ConfigObject().apply { thing[key] = this } else thing // Create the ConfigObject here and don't rely on the model doing it. + key.replace( + ".", + "__" + ), // NOTE: We do this key replacement of 'dots' with double underscores to be able to differentiate between nested objects and JSON object keys that contain 'dots' in them. + if (nextElement is JsonObject) ConfigObject().apply { + thing[key] = this + } else thing // Create the ConfigObject here and don't rely on the model doing it. ) } } diff --git a/src/nativeTest/kotlin/com/zepben/zconf/sources/JsonFileSourceProcessorTest.kt b/src/nativeTest/kotlin/com/zepben/zconf/sources/JsonFileSourceProcessorTest.kt index d2dca5a..acad27d 100644 --- a/src/nativeTest/kotlin/com/zepben/zconf/sources/JsonFileSourceProcessorTest.kt +++ b/src/nativeTest/kotlin/com/zepben/zconf/sources/JsonFileSourceProcessorTest.kt @@ -13,6 +13,9 @@ import com.zepben.zconf.model.ConfigValue import io.kotest.core.spec.style.FunSpec import io.kotest.matchers.shouldBe import kotlinx.cinterop.* +import kotlinx.io.Buffer +import kotlinx.io.files.Path +import kotlinx.io.files.SystemFileSystem import platform.posix.getcwd @OptIn(ExperimentalForeignApi::class) @@ -24,22 +27,218 @@ class JsonFileSourceProcessorTest : FunSpec({ tmp.toKString() } - test("handles non existent file") { - JsonFileSourceProcessor("$currentWorkingDir/fake/directory/fake.json").properties + val fixtures = "$currentWorkingDir/src/nativeTest/resources/fixtures" + val optionalRefFixtures = "$fixtures/optional-ref" + val testConfigDir = "$currentWorkingDir/build/zconf-test-fixtures" + + fun fileRef(absolutePath: String) = "file://$absolutePath" + + fun writeTestConfig(name: String, json: String): String { + val path = Path("$testConfigDir/$name") + path.parent?.let { SystemFileSystem.createDirectories(it) } + SystemFileSystem.sink(path).use { sink -> + val buffer = Buffer().apply { write(json.encodeToByteArray()) } + sink.write(buffer, buffer.size) + } + return path.toString() + } + + fun loadConfig(mainPath: String): ConfigObject = + JsonFileSourceProcessor(mainPath).properties as ConfigObject + + fun ConfigObject.shouldBeEmpty() { + contents().isEmpty() shouldBe true + } + + fun ConfigObject.shouldHavePaths(expected: Map?>) { + expected.forEach { (path, value) -> this[path] shouldBe value } } - test("parses a complex JSON var") { - val config = JsonFileSourceProcessor("$currentWorkingDir/src/nativeTest/resources/fixtures/sample.json").properties as ConfigObject - - config["menu.id"] shouldBe ConfigValue("file") - config["menu.value"] shouldBe ConfigValue("File") - config["menu.popup.menuitem.0.value"] shouldBe ConfigValue("New") - config["menu.popup.v1__kubernetes__zepben__com/node-class"] shouldBe ConfigValue("high-memory") - config["menu.numberThree"] shouldBe ConfigValue(3L) - config["menu.numberThreeString"] shouldBe ConfigValue("3") - config["menu.pi"] shouldBe ConfigValue(3.14) - config["menu.piString"] shouldBe ConfigValue("3.14") - config["menu.isTrue"] shouldBe ConfigValue(true) - config["menu.isTrueString"] shouldBe ConfigValue("true") + context("basic file loading") { + test("handles non existent file") { + loadConfig("$currentWorkingDir/fake/directory/fake.json").shouldBeEmpty() + } + + test("parses a complex JSON var") { + loadConfig("$fixtures/sample.json").shouldHavePaths( + mapOf( + "menu.id" to ConfigValue("file"), + "menu.value" to ConfigValue("File"), + "menu.popup.menuitem.0.value" to ConfigValue("New"), + "menu.popup.v1__kubernetes__zepben__com/node-class" to ConfigValue("high-memory"), + "menu.numberThree" to ConfigValue(3L), + "menu.numberThreeString" to ConfigValue("3"), + "menu.pi" to ConfigValue(3.14), + "menu.piString" to ConfigValue("3.14"), + "menu.isTrue" to ConfigValue(true), + "menu.isTrueString" to ConfigValue("true"), + ), + ) + } + } + + context("\$optionalRef") { + test("inlines optional ref when target exists") { + val path = writeTestConfig( + "present-ref.json", + """ + { + "app": { "name": "parent" }, + "metricsDatabase": { + "$OPTIONAL_REF": "${fileRef("$optionalRefFixtures/ref-target.json")}" + } + } + """.trimIndent(), + ) + val config = loadConfig(path) + + config.shouldHavePaths( + mapOf( + "app.name" to ConfigValue("parent"), + "metricsDatabase.host" to ConfigValue("db.example.com"), + "metricsDatabase.port" to ConfigValue(5432L), + ), + ) + } + + test("omits key when optional ref target is missing") { + val path = writeTestConfig( + "missing-ref.json", + """ + { + "app": { "name": "parent" }, + "metricsDatabase": { + "$OPTIONAL_REF": "${fileRef("$optionalRefFixtures/does-not-exist.json")}" + } + } + """.trimIndent(), + ) + val config = loadConfig(path) + + config.shouldHavePaths( + mapOf( + "app.name" to ConfigValue("parent"), + "metricsDatabase" to null, + ), + ) + } + + test("returns empty config when ref target has invalid JSON") { + val path = writeTestConfig( + "invalid-ref.json", + """ + { + "metricsDatabase": { + "$OPTIONAL_REF": "${fileRef("$optionalRefFixtures/invalid-target.json")}" + } + } + """.trimIndent(), + ) + + loadConfig(path).shouldBeEmpty() + } + + test("removes array element when optional ref target is missing") { + val path = writeTestConfig( + "array-missing-ref.json", + """ + { + "items": [ + { "$OPTIONAL_REF": "${fileRef("$optionalRefFixtures/array-ref-target.json")}" }, + { "$OPTIONAL_REF": "${fileRef("$optionalRefFixtures/missing.json")}" }, + { "static": true } + ] + } + """.trimIndent(), + ) + val config = loadConfig(path) + + config.shouldHavePaths( + mapOf( + "items.0.inlined" to ConfigValue(true), + "items.1.static" to ConfigValue(true), + "items.2.static" to null, // Actually removed missing.json ref + ), + ) + } + + test("returns empty config when optional ref uses non-file scheme") { + val path = writeTestConfig( + "bad-scheme.json", + """ + { + "metricsDatabase": { + "$OPTIONAL_REF": "http://example.com/config.json" + } + } + """.trimIndent(), + ) + + loadConfig(path).shouldBeEmpty() + } + + test("resolves root document from optional ref when target exists") { + val path = writeTestConfig( + "root-present-ref.json", + """ + { + "$OPTIONAL_REF": "${fileRef("$optionalRefFixtures/ref-target.json")}" + } + """.trimIndent(), + ) + val config = loadConfig(path) + + config.shouldHavePaths( + mapOf( + "host" to ConfigValue("db.example.com"), + "port" to ConfigValue(5432L), + ), + ) + } + + test("recursively resolves optional refs") { + val rootPath = "$testConfigDir/root-ref.json" + writeTestConfig( + "root-ref.json", + """ + { + "$OPTIONAL_REF": "${fileRef("$testConfigDir/middle-ref.json")}" + } + """.trimIndent(), + ) + writeTestConfig( + "middle-ref.json", + """ + { + "$OPTIONAL_REF": "${fileRef("$testConfigDir/leaf-ref.json")}" + } + """.trimIndent(), + ) + writeTestConfig( + "leaf-ref.json", + """ + { + "leaf-value": "resolved" + } + """.trimIndent(), + ) + val config = loadConfig(rootPath) + + config["leaf-value"] shouldBe ConfigValue("resolved") + + } + + test("returns empty config when root optional ref target is missing") { + val path = writeTestConfig( + "root-missing-ref.json", + """ + { + "$OPTIONAL_REF": "${fileRef("$optionalRefFixtures/does-not-exist.json")}" + } + """.trimIndent(), + ) + + loadConfig(path).shouldBeEmpty() + } } }) diff --git a/src/nativeTest/resources/fixtures/optional-ref/array-ref-target.json b/src/nativeTest/resources/fixtures/optional-ref/array-ref-target.json new file mode 100644 index 0000000..ea7c061 --- /dev/null +++ b/src/nativeTest/resources/fixtures/optional-ref/array-ref-target.json @@ -0,0 +1,3 @@ +{ + "inlined": true +} diff --git a/src/nativeTest/resources/fixtures/optional-ref/invalid-target.json b/src/nativeTest/resources/fixtures/optional-ref/invalid-target.json new file mode 100644 index 0000000..8cf1bc3 --- /dev/null +++ b/src/nativeTest/resources/fixtures/optional-ref/invalid-target.json @@ -0,0 +1 @@ +{ not valid json diff --git a/src/nativeTest/resources/fixtures/optional-ref/ref-leaf.json b/src/nativeTest/resources/fixtures/optional-ref/ref-leaf.json new file mode 100644 index 0000000..5d6df14 --- /dev/null +++ b/src/nativeTest/resources/fixtures/optional-ref/ref-leaf.json @@ -0,0 +1,3 @@ +{ + "deep": "value" +} diff --git a/src/nativeTest/resources/fixtures/optional-ref/ref-target.json b/src/nativeTest/resources/fixtures/optional-ref/ref-target.json new file mode 100644 index 0000000..e224547 --- /dev/null +++ b/src/nativeTest/resources/fixtures/optional-ref/ref-target.json @@ -0,0 +1,4 @@ +{ + "host": "db.example.com", + "port": 5432 +}