[DEV-9353] Support $optionalRef syntax JsonFileSourceProcessor

src/nativeMain/kotlin/com/zepben/zconf/sources/EnvBlobSourceProcessor.kt

@@ -29,7 +29,6 @@ open class EnvBlobSourceProcessor @OptIn(ExperimentalForeignApi::class) construc
     @OptIn(ExperimentalEncodingApi::class)
     override fun execute(): ConfigElement {
         val envValue = envFetcher(input) ?: return ConfigObject()
-
         val json = try {
             val decodedValue = Base64.Default.decode(envValue)
             Json.Default.parseToJsonElement(postProcessEnv(decodedValue))

src/nativeMain/kotlin/com/zepben/zconf/sources/JsonFileSourceProcessor.kt

@@ -8,6 +8,8 @@
 
 package com.zepben.zconf.sources
 
+import kotlinx.io.files.SystemFileSystem
+
 import com.zepben.zconf.model.CompositeConfig
 import com.zepben.zconf.model.ConfigArray
 import com.zepben.zconf.model.ConfigElement
@@ -19,6 +21,8 @@ import kotlinx.io.files.SystemFileSystem
 import kotlinx.io.readString
 import kotlinx.serialization.json.*
 
+const val OPTIONAL_REF = "\$optionalRef"
+
 open class JsonFileSourceProcessor(input: String) : SourceProcessor(input) {
 
     private val logger = KotlinLogging.logger {}
@@ -27,43 +31,108 @@ open class JsonFileSourceProcessor(input: String) : SourceProcessor(input) {
         try {
             val contents = SystemFileSystem.source(Path(input)).buffered().readString()
             val json = Json.Default.decodeFromString<JsonObject>(contents)
+            val resolvedJson = resolveFileRefs(json, baseFile = Path(input))
+                ?: throw Exception("Failed to resolve root level $OPTIONAL_REF: $json")
+
+            return convertToIntermediateForm(resolvedJson)
 
-            return convertToIntermediateForm(json)
         } catch (e: Exception) {
             logger.error(e) { "Failed to read JSON file at $input.. skipping.." }
         }
-
         return ConfigObject()
     }
 
     protected fun convertToIntermediateForm(json: JsonElement): ConfigObject {
         require(json is JsonObject)
-
         val accumulator = ConfigObject() // just assume that we always return a full json document
         convertToIntermediateForm(json, "", accumulator)
         return accumulator
     }
 
+    private fun resolveFileRefs(element: JsonElement, baseFile: Path): JsonElement? {
+        return when (element) {
+            is JsonObject -> {
+                /*
+                 Handle $optionalRefs. The semantics work as follows. Given the following:
+                      "metricsDatabase": {
+                        "$optionalRef": "file:///path/to/metrics/config.json"
+                      }
+
+                 In the case that the config.json exists and is the valid JSON, "metricsDatabase" will refernce it.
+                 In the case that it does not exist, "metricsDatabase" key will be removed.
+                 In the case that it exists but is not valid JSON, then we log an error and fail.
+                 */
+
+                val ref = element[OPTIONAL_REF]?.jsonPrimitive?.contentOrNull
+                if (ref != null) {
+                    require(ref.startsWith("file://")) {
+                        "$OPTIONAL_REF must use file:// scheme at $baseFile, got: $ref"
+                    }
+
+                    val refPath = Path(ref.removePrefix("file://"))
+
+                    if (SystemFileSystem.metadataOrNull(refPath)?.isRegularFile != true) {
+                        logger.debug { "Skipping $OPTIONAL_REF to missing file: $refPath" }
+                        return null
+                    }
+
+                    val contents = SystemFileSystem.source(refPath).buffered().readString()
+                    val resolved = try {
+                        Json.Default.parseToJsonElement(contents)
+                    } catch (e: Exception) {
+                        throw IllegalArgumentException(
+                            "Invalid JSON in $OPTIONAL_REF target '$refPath' (referenced from $baseFile)",
+                            e,
+                        )
+                    }
+                    return resolveFileRefs(resolved, refPath)
+                }
+
+                JsonObject(
+                    element.mapNotNull { (k, v) ->
+                        resolveFileRefs(v, baseFile)?.let { k to it }
+                    }.toMap()
+                )
+            }
+
+            is JsonArray -> JsonArray(
+                element.mapNotNull { resolveFileRefs(it, baseFile) }
+            )
+
+            else -> element
+        }
+    }
+
     private fun convertToIntermediateForm(json: JsonElement, path: String, thing: CompositeConfig) {
         when (json) {
             is JsonNull -> return // We don't care about nulls
             is JsonPrimitive -> thing[path.removePrefix(".")] = json.toKotlinValue()
-            is JsonArray ->{
-                val arr = ConfigArray().apply { thing[path] = this } // Create the ConfigArray here and don't rely on the model doing it.
+            is JsonArray -> {
+                val arr = ConfigArray().apply {
+                    thing[path] = this
+                } // Create the ConfigArray here and don't rely on the model doing it.
                 json.forEachIndexed { index, nextElement ->
-                    val (newPath, obj) = if(nextElement is JsonObject)
-                        "" to ConfigObject().apply { arr[index.toString()] = this } // If the nextElement is a JsonObject, we create a new ConfigObject and assign it to the array at the index.
+                    val (newPath, obj) = if (nextElement is JsonObject)
+                        "" to ConfigObject().apply {
+                            arr[index.toString()] = this
+                        } // If the nextElement is a JsonObject, we create a new ConfigObject and assign it to the array at the index.
                     else
                         index.toString() to arr
 
                     convertToIntermediateForm(nextElement, newPath, obj)
                 }
             }
+
             is JsonObject -> json.entries.forEach { (key, nextElement) ->
                 convertToIntermediateForm(
                     nextElement,
-                    key.replace(".", "__"), // NOTE: We do this key replacement of 'dots' with double underscores to be able to differentiate between nested objects and JSON object keys that contain 'dots' in them.
-                    if(nextElement is JsonObject) ConfigObject().apply { thing[key] = this } else thing // Create the ConfigObject here and don't rely on the model doing it.
+                    key.replace(
+                        ".",
+                        "__"
+                    ), // NOTE: We do this key replacement of 'dots' with double underscores to be able to differentiate between nested objects and JSON object keys that contain 'dots' in them.
+                    if (nextElement is JsonObject) ConfigObject().apply {
+                        thing[key] = this
+                    } else thing // Create the ConfigObject here and don't rely on the model doing it.
                 )
             }
         }

src/nativeTest/kotlin/com/zepben/zconf/sources/JsonFileSourceProcessorTest.kt

@@ -13,6 +13,9 @@ import com.zepben.zconf.model.ConfigValue
 import io.kotest.core.spec.style.FunSpec
 import io.kotest.matchers.shouldBe
 import kotlinx.cinterop.*
+import kotlinx.io.Buffer
+import kotlinx.io.files.Path
+import kotlinx.io.files.SystemFileSystem
 import platform.posix.getcwd
 
 @OptIn(ExperimentalForeignApi::class)
@@ -24,22 +27,218 @@ class JsonFileSourceProcessorTest : FunSpec({
         tmp.toKString()
     }
 
-    test("handles non existent file") {
-        JsonFileSourceProcessor("$currentWorkingDir/fake/directory/fake.json").properties
+    val fixtures = "$currentWorkingDir/src/nativeTest/resources/fixtures"
+    val optionalRefFixtures = "$fixtures/optional-ref"
+    val testConfigDir = "$currentWorkingDir/build/zconf-test-fixtures"
+
+    fun fileRef(absolutePath: String) = "file://$absolutePath"
+
+    fun writeTestConfig(name: String, json: String): String {
+        val path = Path("$testConfigDir/$name")
+        path.parent?.let { SystemFileSystem.createDirectories(it) }
+        SystemFileSystem.sink(path).use { sink ->
+            val buffer = Buffer().apply { write(json.encodeToByteArray()) }
+            sink.write(buffer, buffer.size)
+        }
+        return path.toString()
+    }
+
+    fun loadConfig(mainPath: String): ConfigObject =
+        JsonFileSourceProcessor(mainPath).properties as ConfigObject
+
+    fun ConfigObject.shouldBeEmpty() {
+        contents().isEmpty() shouldBe true
+    }
+
+    fun ConfigObject.shouldHavePaths(expected: Map<String, ConfigValue<*>?>) {
+        expected.forEach { (path, value) -> this[path] shouldBe value }
     }
 
-    test("parses a complex JSON var") {
-        val config = JsonFileSourceProcessor("$currentWorkingDir/src/nativeTest/resources/fixtures/sample.json").properties as ConfigObject
-
-        config["menu.id"] shouldBe ConfigValue("file")
-        config["menu.value"] shouldBe ConfigValue("File")
-        config["menu.popup.menuitem.0.value"] shouldBe ConfigValue("New")
-        config["menu.popup.v1__kubernetes__zepben__com/node-class"] shouldBe ConfigValue("high-memory")
-        config["menu.numberThree"] shouldBe ConfigValue(3L)
-        config["menu.numberThreeString"] shouldBe ConfigValue("3")
-        config["menu.pi"] shouldBe ConfigValue(3.14)
-        config["menu.piString"] shouldBe ConfigValue("3.14")
-        config["menu.isTrue"] shouldBe ConfigValue(true)
-        config["menu.isTrueString"] shouldBe ConfigValue("true")
+    context("basic file loading") {
+        test("handles non existent file") {
+            loadConfig("$currentWorkingDir/fake/directory/fake.json").shouldBeEmpty()
+        }
+
+        test("parses a complex JSON var") {
+            loadConfig("$fixtures/sample.json").shouldHavePaths(
+                mapOf(
+                    "menu.id" to ConfigValue("file"),
+                    "menu.value" to ConfigValue("File"),
+                    "menu.popup.menuitem.0.value" to ConfigValue("New"),
+                    "menu.popup.v1__kubernetes__zepben__com/node-class" to ConfigValue("high-memory"),
+                    "menu.numberThree" to ConfigValue(3L),
+                    "menu.numberThreeString" to ConfigValue("3"),
+                    "menu.pi" to ConfigValue(3.14),
+                    "menu.piString" to ConfigValue("3.14"),
+                    "menu.isTrue" to ConfigValue(true),
+                    "menu.isTrueString" to ConfigValue("true"),
+                ),
+            )
+        }
+    }
+
+    context("\$optionalRef") {
+        test("inlines optional ref when target exists") {
+            val path = writeTestConfig(
+                "present-ref.json",
+                """
+                {
+                  "app": { "name": "parent" },
+                  "metricsDatabase": {
+                    "$OPTIONAL_REF": "${fileRef("$optionalRefFixtures/ref-target.json")}"
+                  }
+                }
+                """.trimIndent(),
+            )
+            val config = loadConfig(path)
+
+            config.shouldHavePaths(
+                mapOf(
+                    "app.name" to ConfigValue("parent"),
+                    "metricsDatabase.host" to ConfigValue("db.example.com"),
+                    "metricsDatabase.port" to ConfigValue(5432L),
+                ),
+            )
+        }
+
+        test("omits key when optional ref target is missing") {
+            val path = writeTestConfig(
+                "missing-ref.json",
+                """
+                {
+                  "app": { "name": "parent" },
+                  "metricsDatabase": {
+                    "$OPTIONAL_REF": "${fileRef("$optionalRefFixtures/does-not-exist.json")}"
+                  }
+                }
+                """.trimIndent(),
+            )
+            val config = loadConfig(path)
+
+            config.shouldHavePaths(
+                mapOf(
+                    "app.name" to ConfigValue("parent"),
+                    "metricsDatabase" to null,
+                ),
+            )
+        }
+
+        test("returns empty config when ref target has invalid JSON") {
+            val path = writeTestConfig(
+                "invalid-ref.json",
+                """
+                {
+                  "metricsDatabase": {
+                    "$OPTIONAL_REF": "${fileRef("$optionalRefFixtures/invalid-target.json")}"
+                  }
+                }
+                """.trimIndent(),
+            )
+
+            loadConfig(path).shouldBeEmpty()
+        }
+
+        test("removes array element when optional ref target is missing") {
+            val path = writeTestConfig(
+                "array-missing-ref.json",
+                """
+                {
+                  "items": [
+                    { "$OPTIONAL_REF": "${fileRef("$optionalRefFixtures/array-ref-target.json")}" },
+                    { "$OPTIONAL_REF": "${fileRef("$optionalRefFixtures/missing.json")}" },
+                    { "static": true }
+                  ]
+                }
+                """.trimIndent(),
+            )
+            val config = loadConfig(path)
+
+            config.shouldHavePaths(
+                mapOf(
+                    "items.0.inlined" to ConfigValue(true),
+                    "items.1.static" to ConfigValue(true),
+                    "items.2.static" to null, // Actually removed missing.json ref
+                ),
+            )
+        }
+
+        test("returns empty config when optional ref uses non-file scheme") {
+            val path = writeTestConfig(
+                "bad-scheme.json",
+                """
+                {
+                  "metricsDatabase": {
+                    "$OPTIONAL_REF": "http://example.com/config.json"
+                  }
+                }
+                """.trimIndent(),
+            )
+
+            loadConfig(path).shouldBeEmpty()
+        }
+
+        test("resolves root document from optional ref when target exists") {
+            val path = writeTestConfig(
+                "root-present-ref.json",
+                """
+                {
+                  "$OPTIONAL_REF": "${fileRef("$optionalRefFixtures/ref-target.json")}"
+                }
+                """.trimIndent(),
+            )
+            val config = loadConfig(path)
+
+            config.shouldHavePaths(
+                mapOf(
+                    "host" to ConfigValue("db.example.com"),
+                    "port" to ConfigValue(5432L),
+                ),
+            )
+        }
+
+        test("recursively resolves optional refs") {
+            val rootPath = "$testConfigDir/root-ref.json"
+            writeTestConfig(
+                "root-ref.json",
+                """
+                {
+                  "$OPTIONAL_REF": "${fileRef("$testConfigDir/middle-ref.json")}"
+                }
+                """.trimIndent(),
+            )
+            writeTestConfig(
+                "middle-ref.json",
+                """
+                {
+                  "$OPTIONAL_REF": "${fileRef("$testConfigDir/leaf-ref.json")}"
+                }
+                """.trimIndent(),
+            )
+            writeTestConfig(
+                "leaf-ref.json",
+                """
+                {
+                  "leaf-value": "resolved"
+                }
+                """.trimIndent(),
+            )
+            val config = loadConfig(rootPath)
+
+            config["leaf-value"] shouldBe ConfigValue("resolved")
+
+        }
+
+        test("returns empty config when root optional ref target is missing") {
+            val path = writeTestConfig(
+                "root-missing-ref.json",
+                """
+                {
+                  "$OPTIONAL_REF": "${fileRef("$optionalRefFixtures/does-not-exist.json")}"
+                }
+                """.trimIndent(),
+            )
+
+            loadConfig(path).shouldBeEmpty()
+        }
     }
 })

src/nativeTest/resources/fixtures/optional-ref/array-ref-target.json

@@ -0,0 +1,3 @@
+{
+    "inlined": true
+}

src/nativeTest/resources/fixtures/optional-ref/invalid-target.json

@@ -0,0 +1 @@
+{ not valid json

src/nativeTest/resources/fixtures/optional-ref/ref-leaf.json

@@ -0,0 +1,3 @@
+{
+    "deep": "value"
+}