From 2f13a664b3fc4ac6d43776ebc9fdfd388fd1ef99 Mon Sep 17 00:00:00 2001 From: Ray Ozzie Date: Tue, 4 Aug 2026 21:04:02 -0400 Subject: [PATCH 1/2] feat: optional packed J layout to cut JSON memory on small MCUs We are running out of RAM on small MCU hosts, and the JSON object model is where it goes. A parsed document costs 5-9x the size of the text it came from, because every object member is three separate heap allocations -- the node, the key, and the value string -- and each one pays the allocator's per-chunk header and rounding. NOTE_C_STORAGE_OPTIMIZATION packs a member's key and short string value into the node's own allocation, so that member costs ONE allocation instead of three. Members are also reordered to remove alignment padding, taking a node from 48 to 40 bytes on a 32-bit target. valueint and valuenumber are unused on a string node, so their bytes carry the value and then the key; anything that doesn't fit still goes to the heap, and a setter that needs those bytes back evacuates first. Backward compatible: the historical layout is untouched and remains the default, so nothing shifts unless you opt in. Both layouts render byte-identical JSON from one shared parser and printer. CI builds and tests both, crossed with single precision and low memory. Measured on a 32-bit target, newlib dlmalloc (4B header, 8B align, 16B min): JSON object txt obj allocs bytes saved ------------------------------------------------------- --- --- -------- ---------- ----- {"req":"card.version"} 22 2 5 -> 2 152 -> 96 37% {"req":"note.add","file":"data.qo","sync":true} 47 4 11 -> 4 304 -> 200 34% {"req":"hub.set","product":"com.blues.airnote",...} 93 6 17 -> 6 472 -> 312 34% {"req":"note.add","file":"air.qo","body":{6 readings}} 122 10 23 -> 10 736 -> 536 27% {"req":"card.location","status":"GPS updated",...} 91 6 17 -> 6 472 -> 288 39% {"err":"note: no notes available {note-noexist}"} 49 2 5 -> 2 176 -> 120 32% {"req":"env.get","name":"monitor-pump","text":"enabled"} 56 4 13 -> 4 328 -> 192 41% {"device":"dev:864475044204278","sn":"pump-A17",...} 75 4 13 -> 4 336 -> 208 38% {"a":"b","c":"d","e":"f","g":"h","i":"j","k":"l"} 49 7 25 -> 7 584 -> 336 42% ------------------------------------------------------- --- --- -------- ---------- ----- TOTAL 604 45 129 -> 45 3560 -> 2288 36% Best case is many short string members: 25 allocations become 7. Worst case is numeric bodies, where there is no string value to absorb into the node, but the key still packs. Over a 50-document corpus the totals are 38% fewer allocations and 26% less heap, dropping expansion from 9.0x to 6.7x. Under NOTE_C_SINGLE_PRECISION the saving is 16%, not 26%: a float JNUMBER already removes the historical layout's tail padding, so both layouts are 40 bytes and the win comes only from packing. Also fixes two pre-existing printer defects found by comparing output against a pristine build of the base revision. Strings containing more than one escaped control character were truncated and emitted invalid JSON, and formatted+omitempty printing emitted indentation for members it then elided. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Hb7xkJBH7VWvvayVHsNZaA --- .github/workflows/ci.yml | 86 ++ ARCHITECTURE.md | 18 +- CMakeLists.txt | 10 + docs/architecture/architecture.html | 2 +- .../decisions/0002-j-node-storage-layout.md | 42 + n_cjson.c | 1095 ++++++++++++++--- n_cjson.h | 255 +++- n_cjson_helpers.c | 16 +- n_lib.h | 11 + note.h | 12 + scripts/generate_corpus_golden.sh | 144 +++ scripts/run_ab_layout_comparison.sh | 191 +++ scripts/run_unit_tests.sh | 5 + test/CMakeLists.txt | 21 + test/README.md | 26 + test/data/corpus_golden.tsv | 205 +++ test/data/corpus_golden_single.tsv | 205 +++ test/include/j_corpus.hpp | 178 +++ test/include/j_layout_test_support.hpp | 662 ++++++++++ test/src/JLayout_test.cpp | 990 +++++++++++++++ test/src/JPublicApiCompat_test.cpp | 629 ++++++++++ test/src/JReferenceAndFailure_test.cpp | 611 +++++++++ test/src/NoteRequestResponseJSON_test.cpp | 22 +- test/src/j_public_api_compat_c.c | 148 +++ 24 files changed, 5404 insertions(+), 180 deletions(-) create mode 100644 docs/architecture/decisions/0002-j-node-storage-layout.md create mode 100755 scripts/generate_corpus_golden.sh create mode 100755 scripts/run_ab_layout_comparison.sh create mode 100644 test/data/corpus_golden.tsv create mode 100644 test/data/corpus_golden_single.tsv create mode 100644 test/include/j_corpus.hpp create mode 100644 test/include/j_layout_test_support.hpp create mode 100644 test/src/JLayout_test.cpp create mode 100644 test/src/JPublicApiCompat_test.cpp create mode 100644 test/src/JReferenceAndFailure_test.cpp create mode 100644 test/src/j_public_api_compat_c.c diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa31d998..a745f407 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -191,6 +191,92 @@ jobs: run: | docker run --rm --volume $(pwd):/note-c/ --workdir /note-c/ --entrypoint ./scripts/run_unit_tests.sh ghcr.io/blues/note_c_ci:latest --mem-check --low-mem --single-precision + run_layout_matrix_unit_tests: + # The historical J layout is the default; NOTE_C_STORAGE_OPTIMIZATION packs + # keys and short string values into the node's own allocation. Both layouts + # are shipped, so both must stay green. + # + # Precision is crossed in because it is not independent of layout: JNUMBER + # is a float under NOTE_C_SINGLE_PRECISION, which changes the size and tail + # padding of the inline region the packed layout carves its storage out of. + # A defect confined to one cell of this matrix is otherwise easy to miss. + # + # Low memory is crossed in as an independent axis too, because it changes + # which code compiles at all. + # + # Between this matrix and the two standalone jobs, all eight cells of + # {layout} x {precision} x {low-mem} are built and run: default/double is + # run_unit_tests, default/low-mem/single is run_low_mem_unit_tests, and the + # remaining six are below. + runs-on: ubuntu-latest + if: ${{ always() }} + needs: [build_ci_docker_image] + + strategy: + fail-fast: false + matrix: + include: + - name: default layout, single precision + flags: --single-precision + - name: storage optimization, default precision + flags: --storage-optimization + - name: storage optimization, single precision + flags: --storage-optimization --single-precision + - name: storage optimization, low memory, single precision + flags: --storage-optimization --low-mem --single-precision + - name: default layout, low memory, double precision + flags: --low-mem + - name: storage optimization, low memory, double precision + flags: --storage-optimization --low-mem + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Load CI Docker image + # Only load the Docker image artifact if build_ci_docker_image actually + # ran (e.g. it wasn't skipped and was successful). + if: ${{ needs.build_ci_docker_image.result == 'success' }} + uses: ./.github/actions/load-ci-image + + - name: Run tests (${{ matrix.name }}) + run: | + docker run --rm --volume $(pwd):/note-c/ --workdir /note-c/ --entrypoint ./scripts/run_unit_tests.sh ghcr.io/blues/note_c_ci:latest --mem-check ${{ matrix.flags }} + + run_j_layout_ab_comparison: + # Proves the two layouts render byte-identically, and publishes the memory + # difference in the job log. + # + # Run at both precisions. JNUMBER is a float under NOTE_C_SINGLE_PRECISION, + # which removes the historical layout's tail padding on its own: sizeof(J) + # is 40 in BOTH layouts there, so the saving is materially smaller and comes + # entirely from inline packing. Publishing only the double-precision table + # would overstate the win for single-precision targets. + runs-on: ubuntu-latest + if: ${{ always() }} + needs: [build_ci_docker_image] + + strategy: + fail-fast: false + matrix: + include: + - name: double precision + flags: "" + - name: single precision + flags: --single-precision + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Load CI Docker image + if: ${{ needs.build_ci_docker_image.result == 'success' }} + uses: ./.github/actions/load-ci-image + + - name: Compare default and storage-optimized J layouts (${{ matrix.name }}) + run: | + docker run --rm --volume $(pwd):/note-c/ --workdir /note-c/ --entrypoint ./scripts/run_ab_layout_comparison.sh ghcr.io/blues/note_c_ci:latest ${{ matrix.flags }} + run_astyle: runs-on: ubuntu-latest if: ${{ always() }} diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 9be6e6f4..0b95bc1e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -35,7 +35,7 @@ Use this file to understand what the architecture is meant to be. Use `source-re - `n_serial.c`: serial transport implementation and chunked newline-framed serial transmit/receive behavior. - `n_i2c.c`: I2C transport implementation and chunked newline-framed I2C transmit/receive behavior. - `n_hooks.c`: global function-pointer hook registry, active-interface dispatch, and invocation of platform hooks for memory, time, mutexes, debug output, and transports. -- `n_cjson.c`, `n_cjson.h`, `n_cjson_helpers.c`: bundled JSON representation and helper APIs. +- `n_cjson.c`, `n_cjson.h`, `n_cjson_helpers.c`: bundled JSON representation and helper APIs. The `J` node has two compile-time storage layouts; see "JSON node storage" below. - `n_helpers.c`, `n_str.c`, `n_printf.c`, `n_atof.c`, `n_ftoa.c`, `n_b64.c`, `n_cobs.c`, `n_md5.c`, `n_const.c`, `n_ua.c`: portability helpers, encoding, formatting, constants, and utility behavior. - `test/`: unit tests and mocks for protecting SDK behavior without requiring real hardware. - `scripts/`: local automation for checks, documentation, and release support. @@ -68,6 +68,17 @@ Applications normally build requests as `J` objects, send them through `NoteRequ Transport and platform behavior is supplied through hooks so the same core code can run on microcontrollers, embedded Linux, tests, and other C/C++ environments. Serial and I2C transports move raw newline-framed bytes through hook dispatch. Binary payload helpers, not the transport implementations, own COBS framing and MD5 verification. +## JSON Node Storage + +`J` has two storage layouts, selected at compile time. Both produce byte-identical JSON and expose the same API; they differ only in how a node's key and string value are allocated. + +- **Historical layout (default).** A node is 48 bytes on a 32-bit target. A member's key and its string value each occupy their own heap allocation, so `"key":"value"` costs three allocations. +- **`NOTE_C_STORAGE_OPTIMIZATION`.** A node is 40 bytes, and a member's key and short string value are carved out of the node's own allocation, so the same member costs one allocation. The bytes occupied by `valueint` and `valuenumber` — which a `JString`/`JRaw` node never uses — hold that content, and a `objlen` member records the node's allocation size. Over a representative corpus this removes roughly 38% of allocations and 26% of the heap a parsed document holds. + +Ownership is expressed by four `type` flags: `JIsReference` and `JStringIsConst` (pre-existing, unchanged meanings), plus `JValueInline` and `JKeyInline`, which are only set under the optimization and record that a pointer addresses the node's own allocation. Every mutation of `valuestring` and `string` routes through storage helpers in `n_cjson.c`, so the layout-conditional code stays confined to those helpers; the parser, printer, and public API are layout-agnostic. + +The optimization is **off by default**, deferred pending a planned external beta of the historical layout, and because it changes the public representation of `J`: `sizeof(J)` changes, member offsets move, `type` narrows from `int` to `uint16_t`, `objlen` is added, and on a string node `valueint`/`valuenumber` may hold character data. note-c is always compiled from source as part of the customer solution, so the requirement is that every translation unit seeing `J` uses the same setting — the note-c sources and the customer sources alike. The CMake option propagates `PUBLIC` to enforce that, exactly as `NOTE_C_SINGLE_PRECISION` does. See `docs/architecture/decisions/0002-j-node-storage-layout.md`. + ## Public Contracts The main compatibility contracts are: @@ -77,7 +88,8 @@ The main compatibility contracts are: - Request/response ownership semantics for public transaction APIs. - Hook signatures for serial, I2C, memory, mutex, time, and debug output. - Notecard request/response semantics and timeout/retry behavior. -- Build configuration behavior for low-memory, single-precision, user-agent, CRC, and portability-helper variants. +- Build configuration behavior for low-memory, single-precision, storage-optimization, user-agent, CRC, and portability-helper variants. +- The in-memory representation of `J` itself, which downstream code reads directly. `NOTE_C_STORAGE_OPTIMIZATION` changes that representation and is therefore opt-in; see "JSON node storage". - Version constants and release expectations documented in `README.md`. Breaking changes to these contracts require deliberate versioning, migration notes, and architecture documentation updates. @@ -88,6 +100,8 @@ Breaking changes to these contracts require deliberate versioning, migration not Build configuration is part of the portability model. CMake detects platform `strlcpy`/`strlcat` support and only includes bundled `n_str.c` helpers when needed. Low-memory builds disable user-agent support and request CRC paths, omit `n_ua.c`, use compact error/log constants, and reduce allocation chunk size. +Two CMake options change the public representation rather than only behavior, and are propagated `PUBLIC` so every translation unit that sees `J` agrees: `NOTE_C_SINGLE_PRECISION` (which changes the width of `JNUMBER`) and `NOTE_C_STORAGE_OPTIMIZATION` (which changes the layout of `J`). note-c is always compiled from source as part of the consuming solution, so this is a single-build consistency requirement. + ## Runtime Model At runtime, host code initializes the relevant hooks, constructs Notecard requests, and calls `note-c` APIs. `note-c` serializes requests, sends bytes through the selected hook-backed transport, parses responses, and returns JSON objects or status to the caller. diff --git a/CMakeLists.txt b/CMakeLists.txt index 2803711f..5877464c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,13 @@ option(NOTE_NODEBUG "Build the library without debug information." OFF) option(NOTE_C_NO_LIBC "Audit libc dependencies by linking the note_c test/audit library without libc, generating errors for any undefined symbols." OFF) option(NOTE_C_SHOW_MALLOC "Build the library with flags required to log memory usage." OFF) option(NOTE_C_SINGLE_PRECISION "Use single precision for JSON floating point numbers." OFF) +# Opt in to the packed J layout, which carves a member's key and short string +# value out of the node's own allocation instead of giving each its own heap +# block. The historical layout is the default. Like NOTE_C_SINGLE_PRECISION this +# changes sizeof(J), so it is propagated PUBLIC and must match between the +# library and every consumer. See docs/architecture/decisions/ +# 0002-j-node-storage-layout.md. +option(NOTE_C_STORAGE_OPTIMIZATION "Pack J keys and short string values into the node's own allocation (smaller, fewer allocations; changes the public J representation)." OFF) option(NOTE_C_HEARTBEAT_CALLBACK "Enable heartbeat callback support." OFF) # NOTE_C_NO_LIBC is a link-time undefined-symbol audit (see @@ -103,6 +110,9 @@ function(note_c_configure_common target) if(NOTE_C_SINGLE_PRECISION) target_compile_definitions(${target} PUBLIC NOTE_C_SINGLE_PRECISION) endif() + if(NOTE_C_STORAGE_OPTIMIZATION) + target_compile_definitions(${target} PUBLIC NOTE_C_STORAGE_OPTIMIZATION) + endif() if(NOTE_C_HEARTBEAT_CALLBACK) target_compile_definitions(${target} PUBLIC NOTE_C_HEARTBEAT_CALLBACK) endif() diff --git a/docs/architecture/architecture.html b/docs/architecture/architecture.html index 8c7e1dcd..f55305ef 100644 --- a/docs/architecture/architecture.html +++ b/docs/architecture/architecture.html @@ -928,7 +928,7 @@

Nodes

- +