From efaa3089ce1585d38f1d771b79cb0e538eea55a4 Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Fri, 15 May 2026 10:43:55 +0100 Subject: [PATCH 1/2] proto-{flow,gazette,grpc}: regenerate from current prost-build toolchain Mechanical regeneration only; no .proto changes. Captures formatting shifts from the local prost-build/protoc toolchain (e.g. `*` -> `-` bullets in doc comments, multi-line attribute reflow, type-parameter line breaks). Splitting this from the next commit so review can focus on the functional diff. --- crates/proto-flow/src/capture.rs | 32 ++++++++++++---------------- crates/proto-flow/src/flow.rs | 10 ++++----- crates/proto-flow/src/materialize.rs | 25 ++++++++++------------ crates/proto-flow/src/ops.rs | 2 +- crates/proto-flow/src/runtime.rs | 8 +++---- crates/proto-flow/src/shuffle.rs | 9 ++++---- 6 files changed, 38 insertions(+), 48 deletions(-) diff --git a/crates/proto-flow/src/capture.rs b/crates/proto-flow/src/capture.rs index a341193f4fc..f385237c127 100644 --- a/crates/proto-flow/src/capture.rs +++ b/crates/proto-flow/src/capture.rs @@ -201,7 +201,7 @@ pub mod response { /// One or more JSON pointers, which are used to extract resource paths /// from resource configurations of this connector. For example, /// a database connector might have a resource config like: - /// {"schema": "foo", "table": "bar", "other": "config", "answer": 42} + /// {"schema": "foo", "table": "bar", "other": "config", "answer": 42} /// The connector would specify `resource_path_pointers: \["/schema", "/table"\]`, /// which would result in a `resource_path` of `\["foo", "bar"\]`. #[prost(string, repeated, tag = "6")] @@ -240,10 +240,9 @@ pub mod response { pub disable: bool, /// Components of the resource path which fully qualify the resource /// identified by this binding. - /// - /// * For an RDBMS, this might be \[\]{dbname, schema, table}. - /// * For Kafka, this might be \[\]{topic}. - /// * For Redis, this might be \[\]{key_prefix}. + /// - For an RDBMS, this might be \[\]{dbname, schema, table}. + /// - For Kafka, this might be \[\]{topic}. + /// - For Redis, this might be \[\]{key_prefix}. #[prost(string, repeated, tag = "6")] pub resource_path: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, /// Discovered bindings should reflect the natural identifiers of the @@ -256,17 +255,15 @@ pub mod response { /// If the answer is "yes", the key is NOT a fallback. /// /// For regular (non-fallback) keys: - /// - /// * The Estuary control-plane will update collection keys to reflect - /// the latest discovered keys from the source endpoint. - /// * Users should not change such collection keys (they'll be overwritten). + /// - The Estuary control-plane will update collection keys to reflect + /// the latest discovered keys from the source endpoint. + /// - Users should not change such collection keys (they'll be overwritten). /// /// For fallback keys: - /// - /// * The Estuary control-plane will initialize a new collection with the - /// fallback key. - /// * Users may change the collection key, and it will not be overwritten - /// by future discoveries. + /// - The Estuary control-plane will initialize a new collection with the + /// fallback key. + /// - Users may change the collection key, and it will not be overwritten + /// by future discoveries. #[prost(bool, tag = "7")] pub is_fallback_key: bool, } @@ -285,10 +282,9 @@ pub mod response { pub struct Binding { /// Components of the resource path which fully qualify the resource /// identified by this binding. - /// - /// * For an RDBMS, this might be \[\]{dbname, schema, table}. - /// * For Kafka, this might be \[\]{topic}. - /// * For Redis, this might be \[\]{key_prefix}. + /// - For an RDBMS, this might be \[\]{dbname, schema, table}. + /// - For Kafka, this might be \[\]{topic}. + /// - For Redis, this might be \[\]{key_prefix}. #[prost(string, repeated, tag = "1")] pub resource_path: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } diff --git a/crates/proto-flow/src/flow.rs b/crates/proto-flow/src/flow.rs index 3c4040c8a0a..ca3eed75a86 100644 --- a/crates/proto-flow/src/flow.rs +++ b/crates/proto-flow/src/flow.rs @@ -59,8 +59,8 @@ pub struct Projection { #[derive(Clone, PartialEq, ::prost::Message)] pub struct Inference { /// The possible types for this location. - /// Subset of \["null", "boolean", "object", "array", "integer", "numeric", - /// "string"\]. + /// Subset of ["null", "boolean", "object", "array", "integer", "numeric", + /// "string"]. #[prost(string, repeated, tag = "1")] pub types: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, #[prost(message, optional, tag = "3")] @@ -151,8 +151,8 @@ pub mod inference { #[prost(uint32, tag = "3")] pub max_items: u32, /// The possible types of items contained in this array. - /// Subset of \["null", "boolean", "object", "array", "integer", "numeric", - /// "string"\]. + /// Subset of ["null", "boolean", "object", "array", "integer", "numeric", + /// "string"]. #[prost(string, repeated, tag = "4")] pub item_types: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } @@ -521,7 +521,7 @@ pub struct FieldSelection { #[prost(string, tag = "3")] pub document: ::prost::alloc::string::String, /// Additional configuration, keyed by fields included in |keys|, |values|, or - /// \|document|. Values are arbitrary JSON-encoded objects. + /// |document|. Values are arbitrary JSON-encoded objects. #[prost(btree_map = "string, bytes", tag = "4")] pub field_config_json_map: ::prost::alloc::collections::BTreeMap< ::prost::alloc::string::String, diff --git a/crates/proto-flow/src/materialize.rs b/crates/proto-flow/src/materialize.rs index a68e97c8598..7f9caecfe6c 100644 --- a/crates/proto-flow/src/materialize.rs +++ b/crates/proto-flow/src/materialize.rs @@ -136,7 +136,7 @@ pub mod request { /// /// If the remote store is authoritative: /// The driver MUST fence off other streams of this materialization that - /// overlap the provided \[key_begin, key_end) range, such that those streams + /// overlap the provided [key_begin, key_end) range, such that those streams /// cannot issue further commits. The driver MUST return its stored runtime /// checkpoint for this materialization and range \[key_begin, key_end\] /// in its Opened response. @@ -288,10 +288,9 @@ pub mod response { /// JSON schema of the connector's resource configuration. /// Schemas must use specific annotations to communicate the expected /// locations of injected platform variables: - /// - /// * `x-collection-name: true`, a string location for a resource name (required). - /// * `x-schema-name: true`, a string location for a resource schema (optional). - /// * `x-delta-updates: true`, a boolean location for enabling delta-updates mode (optional). + /// - `x-collection-name: true`, a string location for a resource name (required). + /// - `x-schema-name: true`, a string location for a resource schema (optional). + /// - `x-delta-updates: true`, a boolean location for enabling delta-updates mode (optional). #[prost(bytes = "bytes", tag = "3")] pub resource_config_schema_json: ::prost::bytes::Bytes, /// URL for connector's documentation. @@ -325,10 +324,9 @@ pub mod response { /// technical limitations on length, character set, or case sensitivity. /// For these cases, the connector should provide a "folded" field name to /// be used instead. Examples of folds include: - /// - /// * Lower-casing - /// * Replacing `/` with `_` - /// * Mapping Unicode to ASCII via + /// - Lower-casing + /// - Replacing `/` with `_` + /// - Mapping Unicode to ASCII via /// /// Folds may be lossy and result in duplicated folded field values. /// That's okay. The control plane will ensure at most one field is @@ -445,10 +443,9 @@ pub mod response { ::prost::alloc::collections::BTreeMap<::prost::alloc::string::String, Constraint>, /// Components of the resource path which fully qualify the resource /// identified by this binding. - /// - /// * For an RDBMS, this might be \[\]{dbname, schema, table}. - /// * For Kafka, this might be \[\]{topic}. - /// * For Redis, this might be \[\]{key_prefix}. + /// - For an RDBMS, this might be \[\]{dbname, schema, table}. + /// - For Kafka, this might be \[\]{topic}. + /// - For Redis, this might be \[\]{key_prefix}. #[prost(string, repeated, tag = "2")] pub resource_path: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, /// Materialize combined delta updates of documents rather than full @@ -547,7 +544,7 @@ pub mod response { /// On receipt, the runtime may begin to flush, store, and commit a /// next (pipelined) transaction. /// - /// Acknowledged is *not* a direct response to Request.Acknowledge, + /// Acknowledged is _not_ a direct response to Request.Acknowledge, /// and Acknowledge vs Acknowledged may be written in either order. #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct Acknowledged { diff --git a/crates/proto-flow/src/ops.rs b/crates/proto-flow/src/ops.rs index 57116332c6d..4abb13fe18e 100644 --- a/crates/proto-flow/src/ops.rs +++ b/crates/proto-flow/src/ops.rs @@ -264,7 +264,7 @@ pub mod stats { pub uptime_seconds: u32, /// Usage rate adjustment which accompanies and adjusts `uptime_seconds`. /// The effective number of "used" task seconds is: - /// round(uptime_seconds * usage_rate) + /// round(uptime_seconds * usage_rate) /// /// At present, capture and materialization tasks always use a fixed value of 1.0, /// while derivation tasks use a fixed value of 0.0. diff --git a/crates/proto-flow/src/runtime.rs b/crates/proto-flow/src/runtime.rs index b4abfa0eebf..e9173e15c04 100644 --- a/crates/proto-flow/src/runtime.rs +++ b/crates/proto-flow/src/runtime.rs @@ -15,9 +15,8 @@ pub struct TaskServiceConfig { /// ShuffleRequest is the request message of a Shuffle RPC. /// It's a description of a document shuffle, /// where a journal is read and each document is mapped into: -/// -/// * An extracted, packed, and hashed composite key (a "shuffle key"). -/// * A rotated Clock value (an "r-clock"). +/// - An extracted, packed, and hashed composite key (a "shuffle key"). +/// - A rotated Clock value (an "r-clock"). /// /// The packed key and r-clock can then be compared to individual reader /// RangeSpec's. @@ -103,7 +102,7 @@ pub struct ShuffleResponse { #[prost(message, repeated, tag = "7")] pub docs: ::prost::alloc::vec::Vec, /// The journal offsets of each document within the requested journal. - /// For a document at index i, its offsets are \[ offsets\[2*i\], offsets\[2*i+1\] + /// For a document at index i, its offsets are \[ offsets[2*i\], offsets\[2*i+1\] /// ). #[prost(int64, repeated, packed = "false", tag = "8")] pub offsets: ::prost::alloc::vec::Vec, @@ -398,7 +397,6 @@ pub struct CombineResponse { pub values_packed: ::prost::bytes::Bytes, } /// No requests are sent by the client in a ProxyConnectors RPC. However: -/// /// * The client should leave its stream open while the proxy is in use. /// * Then, it sends EOF to begin a graceful stop of the proxy. /// * The response stream will EOF only after all logs have been yielded. diff --git a/crates/proto-flow/src/shuffle.rs b/crates/proto-flow/src/shuffle.rs index 8ff3b4c7574..d29de926cea 100644 --- a/crates/proto-flow/src/shuffle.rs +++ b/crates/proto-flow/src/shuffle.rs @@ -58,9 +58,8 @@ pub struct ProducerFrontier { #[prost(fixed64, tag = "3")] pub hinted_commit: u64, /// Journal byte offset, sign-encoded: - /// - /// * Non-negative: begin offset of first pending CONTINUE_TXN. - /// * Negative: negation of end offset of last committing ACK_TXN / OUTSIDE_TXN. + /// - Non-negative: begin offset of first pending CONTINUE_TXN. + /// - Negative: negation of end offset of last committing ACK_TXN / OUTSIDE_TXN. #[prost(int64, tag = "4")] pub offset: i64, } @@ -72,7 +71,7 @@ pub struct ProducerFrontier { /// However, Producer IDs come and go over time, and the historical set of producers /// who have EVER written to a journal may be large. /// -/// Most JournalFrontier focuses on *deltas* of frontier state, and have few producers. +/// Most JournalFrontier focuses on _deltas_ of frontier state, and have few producers. /// The exception is when a Coordinator client is streaming in a resume checkpoint. #[derive(Clone, PartialEq, ::prost::Message)] pub struct JournalFrontier { @@ -383,7 +382,7 @@ pub mod log_request { #[prost(bytes = "bytes", tag = "10")] pub doc_archived: ::prost::bytes::Bytes, /// Byte length of this document in its source journal (end_offset - begin_offset). - /// Always fits in uint32 because a single document is always \< 4 GiB. + /// Always fits in uint32 because a single document is always < 4 GiB. #[prost(uint32, tag = "11")] pub source_byte_length: u32, } From 223591e796c8e4230d73ca01a240591ceac28588 Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Thu, 14 May 2026 19:07:06 +0100 Subject: [PATCH 2/2] doc/shape: add top-level contentMediaType to all shapes --- crates/assemble/src/lib.rs | 19 +- .../snapshots/assemble__test__inference.snap | 4 + crates/doc/src/shape/inference.rs | 33 +- crates/doc/src/shape/intersect.rs | 4 +- crates/doc/src/shape/mod.rs | 11 +- crates/doc/src/shape/schema.rs | 18 +- ...nference__test__annotation_collection.snap | 8 +- crates/doc/src/shape/union.rs | 3 +- crates/proto-flow/src/flow.rs | 8 + crates/proto-flow/src/flow.serde.rs | 18 + crates/proto-flow/tests/regression.rs | 1 + .../regression__capture_request_json.snap | 9 +- .../regression__capture_request_proto.snap | 257 ++++----- .../regression__capture_spec_json.snap | 3 +- .../regression__capture_spec_proto.snap | 72 +-- .../regression__collection_spec_json.snap | 3 +- .../regression__collection_spec_proto.snap | 36 +- .../regression__derivation_spec_json.snap | 6 +- .../regression__derivation_spec_proto.snap | 156 ++--- .../regression__derive_request_json.snap | 9 +- .../regression__derive_request_proto.snap | 204 +++---- ...regression__materialization_spec_json.snap | 4 +- ...egression__materialization_spec_proto.snap | 111 ++-- .../regression__materialize_request_json.snap | 9 +- ...regression__materialize_request_proto.snap | 381 ++++++------- ...ets__bindings_are_backfilled_on_reset.snap | 8 + ...ackfills_for_disable_task_and_binding.snap | 8 + ...nferred_schema_generation_id_fallback.snap | 8 + ...llection_inferred_schema_is_unchanged.snap | 8 + ...collection_inferred_schema_is_updated.snap | 8 + ...lection_inferred_schema_not_available.snap | 10 + ...tion_inferred_schema_wrong_generation.snap | 10 + ...lection_schema_contains_flow_document.snap | 4 + ...n_is_skipped_when_shards_are_disabled.snap | 6 +- ..._tests__disabled_bindings_are_ignored.snap | 25 +- .../scenario_tests__golden_all_visits.snap | 150 ++++- ...zation_constraints_on_excluded_fields.snap | 13 +- ...tion_not_created_for_empty_properties.snap | 5 + ...on_tests__deletion_of_used_collection.snap | 6 +- ...tion_of_used_collection_when_disabled.snap | 6 +- ...ansition_tests__disable_live_bindings.snap | 15 +- .../transition_tests__disable_shards.snap | 15 +- .../snapshots/transition_tests__inserts.snap | 38 +- ...on_tests__manual_redact_salt_override.snap | 42 +- .../transition_tests__reset_capture.snap | 9 + .../transition_tests__reset_derivation.snap | 23 + ...ansition_tests__reset_materialization.snap | 10 +- ...ition_tests__reset_of_used_collection.snap | 42 +- ..._update_collection_becomes_derivation.snap | 5 + .../snapshots/transition_tests__updates.snap | 42 +- ...sts__updates_with_clobbered_meta_path.snap | 42 +- go/protocols/flow/flow.pb.go | 532 ++++++++++-------- go/protocols/flow/flow.proto | 7 + 53 files changed, 1581 insertions(+), 903 deletions(-) diff --git a/crates/assemble/src/lib.rs b/crates/assemble/src/lib.rs index bf13b930773..c42cd369bc3 100644 --- a/crates/assemble/src/lib.rs +++ b/crates/assemble/src/lib.rs @@ -55,6 +55,12 @@ pub fn inference(shape: &Shape, exists: Exists) -> flow::Inference { shape::Redact::Unset => flow::inference::Redact::Unset, }; + let content_media_type = shape + .content_media_type + .as_deref() + .unwrap_or_default() + .to_string(); + flow::Inference { types: shape.type_.to_vec(), exists: exists as i32, @@ -66,14 +72,12 @@ pub fn inference(shape: &Shape, exists: Exists) -> flow::Inference { .unwrap_or_default(), default_json, secret: shape.secret.unwrap_or_default(), + // `content_media_type` lives at the top level. For back-compat, + // `Inference.String.content_type` (below) carries the same value + // whenever the projection's type includes "string". string: if shape.type_.overlaps(types::STRING) { Some(flow::inference::String { - content_type: shape - .string - .content_type - .clone() - .map(Into::into) - .unwrap_or_default(), + content_type: content_media_type.clone(), format: shape .string .format @@ -135,6 +139,7 @@ pub fn inference(shape: &Shape, exists: Exists) -> flow::Inference { enum_json_vec, reduce: reduce as i32, redact: redact as i32, + content_media_type, } } @@ -601,10 +606,10 @@ mod test { description: Some("the description".into()), title: Some("the title".into()), secret: Some(true), + content_media_type: Some("a/type".into()), string: StringShape { content_encoding: Some("BaSE64".into()), format: Some(json::schema::formats::Format::DateTime), - content_type: Some("a/type".into()), min_length: 10, max_length: Some(123), }, diff --git a/crates/assemble/src/snapshots/assemble__test__inference.snap b/crates/assemble/src/snapshots/assemble__test__inference.snap index d2d8f89ef20..3645841f398 100644 --- a/crates/assemble/src/snapshots/assemble__test__inference.snap +++ b/crates/assemble/src/snapshots/assemble__test__inference.snap @@ -30,6 +30,7 @@ expression: "&[out1, out2, out3, out4]" ], reduce: Sum, redact: Sha256, + content_media_type: "a/type", }, Inference { types: [ @@ -50,6 +51,7 @@ expression: "&[out1, out2, out3, out4]" ], reduce: Sum, redact: Sha256, + content_media_type: "a/type", }, Inference { types: [ @@ -85,6 +87,7 @@ expression: "&[out1, out2, out3, out4]" ], reduce: Sum, redact: Sha256, + content_media_type: "a/type", }, Inference { types: [ @@ -116,5 +119,6 @@ expression: "&[out1, out2, out3, out4]" ], reduce: Sum, redact: Sha256, + content_media_type: "a/type", }, ] diff --git a/crates/doc/src/shape/inference.rs b/crates/doc/src/shape/inference.rs index f616c25290a..a4f1970c50b 100644 --- a/crates/doc/src/shape/inference.rs +++ b/crates/doc/src/shape/inference.rs @@ -219,7 +219,7 @@ impl Shape { shape.string.content_encoding = Some((&**enc).into()); } Annotation::Core(CoreAnnotation::ContentMediaType(mt)) => { - shape.string.content_type = Some((&**mt).into()); + shape.content_media_type = Some((&**mt).into()); } Annotation::Core(CoreAnnotation::Format(format)) => { shape.string.format = Some(*format); @@ -476,6 +476,9 @@ mod test { - reduce: {strategy: firstWriteWins} - redact: {strategy: sha256} - default: john.doe@gmail.com + - oneOf: + - contentMediaType: some/thing + - contentMediaType: some/thing anyOf: - contentEncoding: base64 - type: object # Elided (impossible). @@ -547,9 +550,9 @@ mod test { None, ))), secret: Some(true), + content_media_type: Some("some/thing".into()), string: StringShape { content_encoding: Some("base64".into()), - content_type: Some("some/thing".into()), format: Some(Format::Email), max_length: None, min_length: 0, @@ -559,6 +562,25 @@ mod test { ); } + #[test] + fn test_one_of_branch_elided_by_outer_type_constraint() { + // A `oneOf` branch whose type is incompatible with the outer + // `$ref`-imposed type set must not leak that type into the final + // shape. Here the `null` branch is impossible given the $ref + // restricts to [string, array]. + let shape = shape_from( + r#" + $defs: + aDef: {type: [string, array]} + oneOf: + - type: array + - type: 'null' + $ref: '#/$defs/aDef' + "#, + ); + assert_eq!(shape.type_, types::ARRAY); + } + #[test] fn test_multiple_reduce_and_redact() { infer_test( @@ -742,6 +764,7 @@ mod test { provenance: Inline, default: None, secret: None, + content_media_type: None, annotations: {}, array: ArrayShape { additional_items: None, @@ -760,7 +783,6 @@ mod test { }, string: StringShape { content_encoding: None, - content_type: None, format: None, max_length: None, min_length: 0, @@ -888,6 +910,7 @@ mod test { shape: enum_fixture(json!(["b"])), }], additional_properties: None, + ..ObjShape::new() }, ..Shape::anything() }, @@ -933,6 +956,7 @@ mod test { }], pattern_properties: Vec::new(), additional_properties: Some(Box::new(enum_fixture(json!(["a", "b"])))), + ..ObjShape::new() }, ..Shape::anything() }, @@ -1133,6 +1157,7 @@ mod test { numeric: NumericShape { minimum: Some(5u64.into()), maximum: Some(10u64.into()), + ..NumericShape::new() }, ..Shape::anything() }, @@ -1244,6 +1269,7 @@ mod test { shape: enum_fixture(json!(["c", "d"])), }], additional_properties: Some(Box::new(enum_fixture(json!([1, 2])))), + ..ObjShape::new() }, ..Shape::anything() }, @@ -1276,6 +1302,7 @@ mod test { shape: enum_fixture(json!(["c", "d"])), }], additional_properties: Some(Box::new(Shape::nothing())), + ..ObjShape::new() }, ..Shape::anything() }, diff --git a/crates/doc/src/shape/intersect.rs b/crates/doc/src/shape/intersect.rs index 38af30c3f59..848003efde9 100644 --- a/crates/doc/src/shape/intersect.rs +++ b/crates/doc/src/shape/intersect.rs @@ -39,7 +39,6 @@ impl StringShape { }; StringShape { content_encoding: lhs.content_encoding.or(rhs.content_encoding), - content_type: lhs.content_type.or(rhs.content_type), format: lhs.format.or(rhs.format), min_length: lhs.min_length.max(rhs.min_length), max_length, @@ -256,6 +255,8 @@ impl Shape { let default = lhs.default.or(rhs.default); let secret = lhs.secret.or(rhs.secret); + let content_media_type = lhs.content_media_type.or(rhs.content_media_type); + let mut annotations = rhs.annotations; annotations.extend(lhs.annotations.into_iter()); @@ -298,6 +299,7 @@ impl Shape { provenance, default, secret, + content_media_type, annotations, string, array, diff --git a/crates/doc/src/shape/mod.rs b/crates/doc/src/shape/mod.rs index 2a6f5b48d07..fea62aa7bd0 100644 --- a/crates/doc/src/shape/mod.rs +++ b/crates/doc/src/shape/mod.rs @@ -36,6 +36,11 @@ pub struct Shape { pub default: Option)>>, /// Is this location sensitive? For example, a password or credential. pub secret: Option, + /// Annotated `contentMediaType`. The JSON-Schema specification defines + /// this annotation only for strings; Flow extends it to apply to any + /// type, so it lives at the top level rather than nested in a typed + /// sub-shape. + pub content_media_type: Option>, /// Annotations are any keywords starting with `X-` or `x-`. /// Their keys and values are collected here, without performing any /// normalization of prefix case. Technically both `x-foo` and `X-foo` may be @@ -52,7 +57,6 @@ pub struct Shape { #[derive(Clone, Debug, Eq, PartialEq)] pub struct StringShape { pub content_encoding: Option>, - pub content_type: Option>, pub format: Option, pub max_length: Option, pub min_length: u32, @@ -135,7 +139,6 @@ impl StringShape { pub const fn new() -> Self { Self { content_encoding: None, - content_type: None, format: None, max_length: None, min_length: 0, @@ -187,6 +190,7 @@ impl Shape { provenance: Provenance::Unset, default: None, secret: None, + content_media_type: None, annotations: BTreeMap::new(), array: ArrayShape::new(), numeric: NumericShape::new(), @@ -208,6 +212,7 @@ impl Shape { provenance: Provenance::Inline, default: None, secret: None, + content_media_type: None, annotations: BTreeMap::new(), array: ArrayShape::new(), numeric: NumericShape::new(), @@ -286,7 +291,7 @@ mod test { fn shape_size_regression() { use super::{ArrayShape, ObjShape, Shape, StringShape}; assert_eq!(std::mem::size_of::(), 56); - assert_eq!(std::mem::size_of::(), 48); + assert_eq!(std::mem::size_of::(), 32); assert_eq!(std::mem::size_of::(), 48); assert_eq!(std::mem::size_of::(), 328); } diff --git a/crates/doc/src/shape/schema.rs b/crates/doc/src/shape/schema.rs index e7f3afbae91..d3beeb90018 100644 --- a/crates/doc/src/shape/schema.rs +++ b/crates/doc/src/shape/schema.rs @@ -18,6 +18,7 @@ fn to_sub_schema(shape: Shape) -> Schema { provenance: _, // Not mapped to a schema. default, secret, + content_media_type, annotations, array, numeric, @@ -53,6 +54,16 @@ fn to_sub_schema(shape: Shape) -> Schema { out.insert("default".to_string(), d.0); } + // The JSON-Schema spec defines `contentMediaType` only for strings; + // Flow extends it to apply to any type, so it lives at the top level + // of Shape. + if let Some(ct) = content_media_type { + out.insert( + keywords::CONTENT_MEDIA_TYPE.to_string(), + serde_json::json!(ct), + ); + } + // Object keywords. if type_.overlaps(types::OBJECT) { let ObjShape { @@ -136,7 +147,6 @@ fn to_sub_schema(shape: Shape) -> Schema { if type_.overlaps(types::STRING) { let StringShape { content_encoding, - content_type, format, max_length, min_length, @@ -148,12 +158,6 @@ fn to_sub_schema(shape: Shape) -> Schema { serde_json::json!(encoding), ); } - if let Some(content_type) = content_type { - out.insert( - keywords::CONTENT_MEDIA_TYPE.to_string(), - serde_json::json!(content_type), - ); - } if let Some(f) = format { out.insert("format".to_string(), serde_json::json!(f.to_string())); } diff --git a/crates/doc/src/shape/snapshots/doc__shape__inference__test__annotation_collection.snap b/crates/doc/src/shape/snapshots/doc__shape__inference__test__annotation_collection.snap index 1280d113dca..1b407031255 100644 --- a/crates/doc/src/shape/snapshots/doc__shape__inference__test__annotation_collection.snap +++ b/crates/doc/src/shape/snapshots/doc__shape__inference__test__annotation_collection.snap @@ -12,6 +12,7 @@ Shape { provenance: Inline, default: None, secret: None, + content_media_type: None, annotations: { "x-test-top-level": Bool(true), }, @@ -43,6 +44,7 @@ Shape { provenance: Inline, default: None, secret: None, + content_media_type: None, annotations: { "X-bar-top-level": Bool(true), "x-bar-two": String("twoVal"), @@ -64,7 +66,6 @@ Shape { }, string: StringShape { content_encoding: None, - content_type: None, format: None, max_length: None, min_length: 0, @@ -87,6 +88,7 @@ Shape { provenance: Inline, default: None, secret: None, + content_media_type: None, annotations: { "x-conflicting-ann": String("yes please"), }, @@ -107,7 +109,6 @@ Shape { }, string: StringShape { content_encoding: None, - content_type: None, format: None, max_length: None, min_length: 0, @@ -128,6 +129,7 @@ Shape { provenance: Inline, default: None, secret: None, + content_media_type: None, annotations: { "X-foo-top-level": Bool(false), "x-foo-one": String("oneVal"), @@ -150,7 +152,6 @@ Shape { }, string: StringShape { content_encoding: None, - content_type: None, format: None, max_length: None, min_length: 0, @@ -161,7 +162,6 @@ Shape { }, string: StringShape { content_encoding: None, - content_type: None, format: None, max_length: None, min_length: 0, diff --git a/crates/doc/src/shape/union.rs b/crates/doc/src/shape/union.rs index eeebd9ae934..a2158e31e49 100644 --- a/crates/doc/src/shape/union.rs +++ b/crates/doc/src/shape/union.rs @@ -52,7 +52,6 @@ impl StringShape { StringShape { content_encoding: union_option(lhs.content_encoding, rhs.content_encoding), - content_type: union_option(lhs.content_type, rhs.content_type), format: Self::union_format(lhs.format, rhs.format), max_length, min_length: lhs.min_length.min(rhs.min_length), @@ -270,6 +269,7 @@ impl Shape { let provenance = lhs.provenance.union(rhs.provenance); let default = union_option(lhs.default, rhs.default); let secret = union_option(lhs.secret, rhs.secret); + let content_media_type = union_option(lhs.content_media_type, rhs.content_media_type); // Union of annotations is actually an _intersection_, which yields only // the annotations that are guaranteed to apply at a given location. @@ -319,6 +319,7 @@ impl Shape { provenance, default, secret, + content_media_type, annotations, string, array, diff --git a/crates/proto-flow/src/flow.rs b/crates/proto-flow/src/flow.rs index ca3eed75a86..09d4fc8a472 100644 --- a/crates/proto-flow/src/flow.rs +++ b/crates/proto-flow/src/flow.rs @@ -93,6 +93,13 @@ pub struct Inference { /// Applied `redact` strategy. #[prost(enumeration = "inference::Redact", tag = "13")] pub redact: i32, + /// Annotated `contentMediaType` of the schema. The JSON-Schema spec + /// defines this annotation only for strings; Flow extends it to apply + /// to any type, so it surfaces at the top level. For backward + /// compatibility, `Inference.String.content_type` continues to be populated + /// whenever this projection includes the "string" type. + #[prost(string, tag = "14")] + pub content_media_type: ::prost::alloc::string::String, } /// Nested message and enum types in `Inference`. pub mod inference { @@ -100,6 +107,7 @@ pub mod inference { #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct String { /// Annotated Content-Type when the projection is of "string" type. + /// Deprecated for non-string projections; prefer `Inference.content_media_type`. #[prost(string, tag = "3")] pub content_type: ::prost::alloc::string::String, /// Annotated format when the projection is of "string" type. diff --git a/crates/proto-flow/src/flow.serde.rs b/crates/proto-flow/src/flow.serde.rs index d22ec1efb24..22698a7cf71 100644 --- a/crates/proto-flow/src/flow.serde.rs +++ b/crates/proto-flow/src/flow.serde.rs @@ -2595,6 +2595,9 @@ impl serde::Serialize for Inference { if self.redact != 0 { len += 1; } + if !self.content_media_type.is_empty() { + len += 1; + } let mut struct_ser = serializer.serialize_struct("flow.Inference", len)?; if !self.types.is_empty() { struct_ser.serialize_field("types", &self.types)?; @@ -2640,6 +2643,9 @@ impl serde::Serialize for Inference { .map_err(|_| serde::ser::Error::custom(format!("Invalid variant {}", self.redact)))?; struct_ser.serialize_field("redact", &v)?; } + if !self.content_media_type.is_empty() { + struct_ser.serialize_field("contentMediaType", &self.content_media_type)?; + } struct_ser.end() } } @@ -2664,6 +2670,8 @@ impl<'de> serde::Deserialize<'de> for Inference { "enum", "reduce", "redact", + "content_media_type", + "contentMediaType", ]; #[allow(clippy::enum_variant_names)] @@ -2680,6 +2688,7 @@ impl<'de> serde::Deserialize<'de> for Inference { EnumJsonVec, Reduce, Redact, + ContentMediaType, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -2713,6 +2722,7 @@ impl<'de> serde::Deserialize<'de> for Inference { "enum" | "enum_json_vec" => Ok(GeneratedField::EnumJsonVec), "reduce" => Ok(GeneratedField::Reduce), "redact" => Ok(GeneratedField::Redact), + "contentMediaType" | "content_media_type" => Ok(GeneratedField::ContentMediaType), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -2744,6 +2754,7 @@ impl<'de> serde::Deserialize<'de> for Inference { let mut enum_json_vec__ = None; let mut reduce__ = None; let mut redact__ = None; + let mut content_media_type__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::Types => { @@ -2823,6 +2834,12 @@ impl<'de> serde::Deserialize<'de> for Inference { } redact__ = Some(map_.next_value::()? as i32); } + GeneratedField::ContentMediaType => { + if content_media_type__.is_some() { + return Err(serde::de::Error::duplicate_field("contentMediaType")); + } + content_media_type__ = Some(map_.next_value()?); + } } } Ok(Inference { @@ -2838,6 +2855,7 @@ impl<'de> serde::Deserialize<'de> for Inference { enum_json_vec: enum_json_vec__.unwrap_or_default(), reduce: reduce__.unwrap_or_default(), redact: redact__.unwrap_or_default(), + content_media_type: content_media_type__.unwrap_or_default(), }) } } diff --git a/crates/proto-flow/tests/regression.rs b/crates/proto-flow/tests/regression.rs index 76ad4fe8a00..9eb22905908 100644 --- a/crates/proto-flow/tests/regression.rs +++ b/crates/proto-flow/tests/regression.rs @@ -19,6 +19,7 @@ fn ex_projections() -> Vec { title: "title".to_string(), exists: inference::Exists::Must as i32, secret: false, + content_media_type: "typ".to_string(), string: Some(inference::String { content_encoding: "enc".to_string(), content_type: "typ".to_string(), diff --git a/crates/proto-flow/tests/snapshots/regression__capture_request_json.snap b/crates/proto-flow/tests/snapshots/regression__capture_request_json.snap index b27ecb1e8ef..0970b63417d 100644 --- a/crates/proto-flow/tests/snapshots/regression__capture_request_json.snap +++ b/crates/proto-flow/tests/snapshots/regression__capture_request_json.snap @@ -70,7 +70,8 @@ expression: json_test(msg) "const-two" ], "reduce": "REDUCE_MERGE", - "redact": "REDACT_SHA256" + "redact": "REDACT_SHA256", + "contentMediaType": "typ" } } ], @@ -172,7 +173,8 @@ expression: json_test(msg) "const-two" ], "reduce": "REDUCE_MERGE", - "redact": "REDACT_SHA256" + "redact": "REDACT_SHA256", + "contentMediaType": "typ" } } ], @@ -323,7 +325,8 @@ expression: json_test(msg) "const-two" ], "reduce": "REDUCE_MERGE", - "redact": "REDACT_SHA256" + "redact": "REDACT_SHA256", + "contentMediaType": "typ" } } ], diff --git a/crates/proto-flow/tests/snapshots/regression__capture_request_proto.snap b/crates/proto-flow/tests/snapshots/regression__capture_request_proto.snap index eea3d060ae9..c0107e79754 100644 --- a/crates/proto-flow/tests/snapshots/regression__capture_request_proto.snap +++ b/crates/proto-flow/tests/snapshots/regression__capture_request_proto.snap @@ -6,17 +6,17 @@ expression: proto_test(msg) |6f6e6669 67227d12 2b080712 157b2264| onfig"}.+....{"d 00000010 |6973636f 76657222 3a22636f 6e666967| iscover":"config 00000020 |227d1a10 64697363 6f766572 2f636170| "}..discover/cap 00000030 -|74757265 1a99040a 1076616c 69646174| ture.....validat 00000040 +|74757265 1a9e040a 1076616c 69646174| ture.....validat 00000040 |652f6361 70747572 6510071a 157b2276| e/capture....{"v 00000050 |616c6964 61746522 3a22636f 6e666967| alidate":"config 00000060 -|227d22de 030a157b 22726573 6f757263| "}"....{"resourc 00000070 -|65223a22 636f6e66 6967227d 12c2030a| e":"config"}.... 00000080 +|227d22e3 030a157b 22726573 6f757263| "}"....{"resourc 00000070 +|65223a22 636f6e66 6967227d 12c7030a| e":"config"}.... 00000080 |1161636d 65436f2f 636f6c6c 65637469| .acmeCo/collecti 00000090 |6f6e1a08 2f6b6579 2f6f6e65 1a082f6b| on../key/one../k 000000a0 |65792f74 776f220b 2f5f6d65 74612f75| ey/two"./_meta/u 000000b0 |7569642a 04747970 652a0672 6567696f| uid*.type*.regio 000000c0 -|6e32a001 0a092f6a 736f6e2f 70747212| n2..../json/ptr. 000000d0 -|07612d66 69656c64 20013287 010a0769| .a-field .2....i 000000e0 +|6e32a501 0a092f6a 736f6e2f 70747212| n2..../json/ptr. 000000d0 +|07612d66 69656c64 2001328c 010a0769| .a-field .2....i 000000e0 |6e746567 65720a06 73747269 6e671a13| nteger..string.. 000000f0 |1a037479 70220464 61746530 b9603a03| ..typ".date0.`:. 00000100 |656e6322 05746974 6c652a04 64657363| enc".title*.desc 00000110 @@ -25,126 +25,127 @@ expression: proto_test(msg) |15080a10 01181422 046e756c 6c220769| .......".null".i 00000140 |6e746567 65725a0b 22636f6e 73742d6f| ntegerZ."const-o 00000150 |6e65225a 0b22636f 6e73742d 74776f22| ne"Z."const-two" 00000160 -|60066803 3a0e7b22 61636b22 3a227472| `.h.:.{"ack":"tr 00000170 -|7565227d 42127b22 77726974 65223a22| ue"}B.{"write":" 00000180 -|73636865 6d61227d 4aa3010a 12706172| schema"}J....par 00000190 -|74697469 6f6e2f74 656d706c 61746510| tition/template. 000001a0 -|031a400a 1e0a0f65 73747561 72792e64| ..@....estuary.d 000001b0 -|65762f66 6f6f120b 6c616265 6c2d7661| ev/foo..label-va 000001c0 -|6c75650a 1e0a0f65 73747561 72792e64| lue....estuary.d 000001d0 -|65762f62 6172120b 6f746865 722d7661| ev/bar..other-va 000001e0 -|6c756522 4208e9ec 0610031a 1273333a| lue"B........s3: 000001f0 -|2f2f6275 636b6574 2f707265 66697822| //bucket/prefix" 00000200 -|0308ac02 3207083e 1080e59a 773a1850| ....2..>....w:.P 00000210 -|6174687b 7b506f73 74666978 2e54656d| ath{{Postfix.Tem 00000220 -|706c6174 657d7d30 0438cfb0 f5015a11| plate}}0.8....Z. 00000230 -|7b227265 6164223a 22736368 656d6122| {"read":"schema" 00000240 -|7d180132 0b31313a 32323a33 333a3434| }..2.11:22:33:44 00000250 -|22b5060a 98060a0e 61636d65 436f2f63| ".......acmeCo/c 00000260 -|61707475 72651007 1a197b22 63617074| apture....{"capt 00000270 -|75726522 3a7b2263 6f6e6669 67223a34| ure":{"config":4 00000280 -|327d7d22 82040a15 7b227265 736f7572| 2}}"....{"resour 00000290 -|6365223a 22636f6e 66696722 7d120473| ce":"config"}..s 000002a0 -|6f6d6512 04706174 681ac203 0a116163| ome..path.....ac 000002b0 -|6d65436f 2f636f6c 6c656374 696f6e1a| meCo/collection. 000002c0 -|082f6b65 792f6f6e 651a082f 6b65792f| ./key/one../key/ 000002d0 -|74776f22 0b2f5f6d 6574612f 75756964| two"./_meta/uuid 000002e0 -|2a047479 70652a06 72656769 6f6e32a0| *.type*.region2. 000002f0 -|010a092f 6a736f6e 2f707472 1207612d| .../json/ptr..a- 00000300 -|6669656c 64200132 87010a07 696e7465| field .2....inte 00000310 -|6765720a 06737472 696e671a 131a0374| ger..string....t 00000320 -|79702204 64617465 30b9603a 03656e63| yp".date0.`:.enc 00000330 -|22057469 746c652a 04646573 63320e7b| ".title*.desc2.{ 00000340 -|22646566 223a2261 756c7422 7d40014a| "def":"ault"}@.J 00000350 -|0b080111 00000000 00408fc0 5215080a| .........@..R... 00000360 -|10011814 22046e75 6c6c2207 696e7465| ....".null".inte 00000370 -|6765725a 0b22636f 6e73742d 6f6e6522| gerZ."const-one" 00000380 -|5a0b2263 6f6e7374 2d74776f 22600668| Z."const-two"`.h 00000390 -|033a0e7b 2261636b 223a2274 72756522| .:.{"ack":"true" 000003a0 -|7d42127b 22777269 7465223a 22736368| }B.{"write":"sch 000003b0 -|656d6122 7d4aa301 0a127061 72746974| ema"}J....partit 000003c0 -|696f6e2f 74656d70 6c617465 10031a40| ion/template...@ 000003d0 -|0a1e0a0f 65737475 6172792e 6465762f| ....estuary.dev/ 000003e0 -|666f6f12 0b6c6162 656c2d76 616c7565| foo..label-value 000003f0 -|0a1e0a0f 65737475 6172792e 6465762f| ....estuary.dev/ 00000400 -|62617212 0b6f7468 65722d76 616c7565| bar..other-value 00000410 -|224208e9 ec061003 1a127333 3a2f2f62| "B........s3://b 00000420 -|75636b65 742f7072 65666978 220308ac| ucket/prefix"... 00000430 -|02320708 3e1080e5 9a773a18 50617468| .2..>....w:.Path 00000440 -|7b7b506f 73746669 782e5465 6d706c61| {{Postfix.Templa 00000450 -|74657d7d 300438cf b0f5015a 117b2272| te}}0.8....Z.{"r 00000460 -|65616422 3a227363 68656d61 227d2003| ead":"schema"} . 00000470 -|2a166125 32466364 63253246 7461626c| *.a%2Fcdc%2Ftabl 00000480 -|652b6261 7a2e7633 28ac0232 80010a0e| e+baz.v3(..2.... 00000490 -|73686172 642f7465 6d706c61 74651a0f| shard/template.. 000004a0 -|7265636f 76657279 2f707265 66697822| recovery/prefix" 000004b0 -|0b68696e 742f7072 65666978 28033202| .hint/prefix(.2. 000004c0 -|083c4801 52400a1e 0a0f6573 74756172| .... 00000770 -|9a773a18 50617468 7b7b506f 73746669| .w:.Path{{Postfi 00000780 -|782e5465 6d706c61 74657d7d 300438cf| x.Template}}0.8. 00000790 -|b0f5015a 117b2272 65616422 3a227363| ...Z.{"read":"sc 000007a0 -|68656d61 227d2003 2a166125 32466364| hema"} .*.a%2Fcd 000007b0 -|63253246 7461626c 652b6261 7a2e7633| c%2Ftable+baz.v3 000007c0 -|28ac0232 80010a0e 73686172 642f7465| (..2....shard/te 000007d0 -|6d706c61 74651a0f 7265636f 76657279| mplate..recovery 000007e0 -|2f707265 66697822 0b68696e 742f7072| /prefix".hint/pr 000007f0 -|65666978 28033202 083c4801 52400a1e| efix(.2..... 00000210 +|9a773a18 50617468 7b7b506f 73746669| .w:.Path{{Postfi 00000220 +|782e5465 6d706c61 74657d7d 300438cf| x.Template}}0.8. 00000230 +|b0f5015a 117b2272 65616422 3a227363| ...Z.{"read":"sc 00000240 +|68656d61 227d1801 320b3131 3a32323a| hema"}..2.11:22: 00000250 +|33333a34 3422ba06 0a9d060a 0e61636d| 33:44".......acm 00000260 +|65436f2f 63617074 75726510 071a197b| eCo/capture....{ 00000270 +|22636170 74757265 223a7b22 636f6e66| "capture":{"conf 00000280 +|6967223a 34327d7d 2287040a 157b2272| ig":42}}"....{"r 00000290 +|65736f75 72636522 3a22636f 6e666967| esource":"config 000002a0 +|227d1204 736f6d65 12047061 74681ac7| "}..some..path.. 000002b0 +|030a1161 636d6543 6f2f636f 6c6c6563| ...acmeCo/collec 000002c0 +|74696f6e 1a082f6b 65792f6f 6e651a08| tion../key/one.. 000002d0 +|2f6b6579 2f74776f 220b2f5f 6d657461| /key/two"./_meta 000002e0 +|2f757569 642a0474 7970652a 06726567| /uuid*.type*.reg 000002f0 +|696f6e32 a5010a09 2f6a736f 6e2f7074| ion2..../json/pt 00000300 +|72120761 2d666965 6c642001 328c010a| r..a-field .2... 00000310 +|07696e74 65676572 0a067374 72696e67| .integer..string 00000320 +|1a131a03 74797022 04646174 6530b960| ....typ".date0.` 00000330 +|3a03656e 63220574 69746c65 2a046465| :.enc".title*.de 00000340 +|7363320e 7b226465 66223a22 61756c74| sc2.{"def":"ault 00000350 +|227d4001 4a0b0801 11000000 0000408f| "}@.J.........@. 00000360 +|c0521508 0a100118 1422046e 756c6c22| .R.......".null" 00000370 +|07696e74 65676572 5a0b2263 6f6e7374| .integerZ."const 00000380 +|2d6f6e65 225a0b22 636f6e73 742d7477| -one"Z."const-tw 00000390 +|6f226006 68037203 7479703a 0e7b2261| o"`.h.r.typ:.{"a 000003a0 +|636b223a 22747275 65227d42 127b2277| ck":"true"}B.{"w 000003b0 +|72697465 223a2273 6368656d 61227d4a| rite":"schema"}J 000003c0 +|a3010a12 70617274 6974696f 6e2f7465| ....partition/te 000003d0 +|6d706c61 74651003 1a400a1e 0a0f6573| mplate...@....es 000003e0 +|74756172 792e6465 762f666f 6f120b6c| tuary.dev/foo..l 000003f0 +|6162656c 2d76616c 75650a1e 0a0f6573| abel-value....es 00000400 +|74756172 792e6465 762f6261 72120b6f| tuary.dev/bar..o 00000410 +|74686572 2d76616c 75652242 08e9ec06| ther-value"B.... 00000420 +|10031a12 73333a2f 2f627563 6b65742f| ....s3://bucket/ 00000430 +|70726566 69782203 08ac0232 07083e10| prefix"....2..>. 00000440 +|80e59a77 3a185061 74687b7b 506f7374| ...w:.Path{{Post 00000450 +|6669782e 54656d70 6c617465 7d7d3004| fix.Template}}0. 00000460 +|38cfb0f5 015a117b 22726561 64223a22| 8....Z.{"read":" 00000470 +|73636865 6d61227d 20032a16 61253246| schema"} .*.a%2F 00000480 +|63646325 32467461 626c652b 62617a2e| cdc%2Ftable+baz. 00000490 +|763328ac 02328001 0a0e7368 6172642f| v3(..2....shard/ 000004a0 +|74656d70 6c617465 1a0f7265 636f7665| template..recove 000004b0 +|72792f70 72656669 78220b68 696e742f| ry/prefix".hint/ 000004c0 +|70726566 69782803 3202083c 48015240| prefix(.2...... 00000780 +|773a1850 6174687b 7b506f73 74666978| w:.Path{{Postfix 00000790 +|2e54656d 706c6174 657d7d30 0438cfb0| .Template}}0.8.. 000007a0 +|f5015a11 7b227265 6164223a 22736368| ..Z.{"read":"sch 000007b0 +|656d6122 7d20032a 16612532 46636463| ema"} .*.a%2Fcdc 000007c0 +|25324674 61626c65 2b62617a 2e763328| %2Ftable+baz.v3( 000007d0 +|ac023280 010a0e73 68617264 2f74656d| ..2....shard/tem 000007e0 +|706c6174 651a0f72 65636f76 6572792f| plate..recovery/ 000007f0 +|70726566 6978220b 68696e74 2f707265| prefix".hint/pre 00000800 +|66697828 03320208 3c480152 400a1e0a| fix(.2... 000001d0 -|80e59a77 3a185061 74687b7b 506f7374| ...w:.Path{{Post 000001e0 -|6669782e 54656d70 6c617465 7d7d3004| fix.Template}}0. 000001f0 -|38cfb0f5 015a117b 22726561 64223a22| 8....Z.{"read":" 00000200 -|73636865 6d61227d 20032a16 61253246| schema"} .*.a%2F 00000210 -|63646325 32467461 626c652b 62617a2e| cdc%2Ftable+baz. 00000220 -|763328ac 02328001 0a0e7368 6172642f| v3(..2....shard/ 00000230 -|74656d70 6c617465 1a0f7265 636f7665| template..recove 00000240 -|72792f70 72656669 78220b68 696e742f| ry/prefix".hint/ 00000250 -|70726566 69782803 3202083c 48015240| prefix(.2......w:.Path{ 000001e0 +|7b506f73 74666978 2e54656d 706c6174| {Postfix.Templat 000001f0 +|657d7d30 0438cfb0 f5015a11 7b227265| e}}0.8....Z.{"re 00000200 +|6164223a 22736368 656d6122 7d20032a| ad":"schema"} .* 00000210 +|16612532 46636463 25324674 61626c65| .a%2Fcdc%2Ftable 00000220 +|2b62617a 2e763328 ac023280 010a0e73| +baz.v3(..2....s 00000230 +|68617264 2f74656d 706c6174 651a0f72| hard/template..r 00000240 +|65636f76 6572792f 70726566 6978220b| ecovery/prefix". 00000250 +|68696e74 2f707265 66697828 03320208| hint/prefix(.2.. 00000260 +|3c480152 400a1e0a 0f657374 75617279| ....w:. 00000180 -|50617468 7b7b506f 73746669 782e5465| Path{{Postfix.Te 00000190 -|6d706c61 74657d7d 300438cf b0f5015a| mplate}}0.8....Z 000001a0 -|117b2272 65616422 3a227363 68656d61| .{"read":"schema 000001b0 -|227d| "} 000001c0 - 000001c2 +|22600668 03720374 79703a0e 7b226163| "`.h.r.typ:.{"ac 000000e0 +|6b223a22 74727565 227d4212 7b227772| k":"true"}B.{"wr 000000f0 +|69746522 3a227363 68656d61 227d4aa3| ite":"schema"}J. 00000100 +|010a1270 61727469 74696f6e 2f74656d| ...partition/tem 00000110 +|706c6174 6510031a 400a1e0a 0f657374| plate...@....est 00000120 +|75617279 2e646576 2f666f6f 120b6c61| uary.dev/foo..la 00000130 +|62656c2d 76616c75 650a1e0a 0f657374| bel-value....est 00000140 +|75617279 2e646576 2f626172 120b6f74| uary.dev/bar..ot 00000150 +|6865722d 76616c75 65224208 e9ec0610| her-value"B..... 00000160 +|031a1273 333a2f2f 6275636b 65742f70| ...s3://bucket/p 00000170 +|72656669 78220308 ac023207 083e1080| refix"....2..>.. 00000180 +|e59a773a 18506174 687b7b50 6f737466| ..w:.Path{{Postf 00000190 +|69782e54 656d706c 6174657d 7d300438| ix.Template}}0.8 000001a0 +|cfb0f501 5a117b22 72656164 223a2273| ....Z.{"read":"s 000001b0 +|6368656d 61227d| chema"} 000001c0 + 000001c7 diff --git a/crates/proto-flow/tests/snapshots/regression__derivation_spec_json.snap b/crates/proto-flow/tests/snapshots/regression__derivation_spec_json.snap index 81fd137731b..6f928aeade4 100644 --- a/crates/proto-flow/tests/snapshots/regression__derivation_spec_json.snap +++ b/crates/proto-flow/tests/snapshots/regression__derivation_spec_json.snap @@ -53,7 +53,8 @@ expression: json_test(msg) "const-two" ], "reduce": "REDUCE_MERGE", - "redact": "REDACT_SHA256" + "redact": "REDACT_SHA256", + "contentMediaType": "typ" } } ], @@ -143,7 +144,8 @@ expression: json_test(msg) "const-two" ], "reduce": "REDUCE_MERGE", - "redact": "REDACT_SHA256" + "redact": "REDACT_SHA256", + "contentMediaType": "typ" } } ], diff --git a/crates/proto-flow/tests/snapshots/regression__derivation_spec_proto.snap b/crates/proto-flow/tests/snapshots/regression__derivation_spec_proto.snap index 0087ea76d37..72c861a9d2a 100644 --- a/crates/proto-flow/tests/snapshots/regression__derivation_spec_proto.snap +++ b/crates/proto-flow/tests/snapshots/regression__derivation_spec_proto.snap @@ -6,8 +6,8 @@ expression: proto_test(msg) |696f6e1a 082f6b65 792f6f6e 651a082f| ion../key/one../ 00000010 |6b65792f 74776f22 0b2f5f6d 6574612f| key/two"./_meta/ 00000020 |75756964 2a047479 70652a06 72656769| uuid*.type*.regi 00000030 -|6f6e32a0 010a092f 6a736f6e 2f707472| on2..../json/ptr 00000040 -|1207612d 6669656c 64200132 87010a07| ..a-field .2.... 00000050 +|6f6e32a5 010a092f 6a736f6e 2f707472| on2..../json/ptr 00000040 +|1207612d 6669656c 64200132 8c010a07| ..a-field .2.... 00000050 |696e7465 6765720a 06737472 696e671a| integer..string. 00000060 |131a0374 79702204 64617465 30b9603a| ...typ".date0.`: 00000070 |03656e63 22057469 746c652a 04646573| .enc".title*.des 00000080 @@ -16,79 +16,79 @@ expression: proto_test(msg) |5215080a 10011814 22046e75 6c6c2207| R.......".null". 000000b0 |696e7465 6765725a 0b22636f 6e73742d| integerZ."const- 000000c0 |6f6e6522 5a0b2263 6f6e7374 2d74776f| one"Z."const-two 000000d0 -|22600668 033a0e7b 2261636b 223a2274| "`.h.:.{"ack":"t 000000e0 -|72756522 7d42127b 22777269 7465223a| rue"}B.{"write": 000000f0 -|22736368 656d6122 7d4aa301 0a127061| "schema"}J....pa 00000100 -|72746974 696f6e2f 74656d70 6c617465| rtition/template 00000110 -|10031a40 0a1e0a0f 65737475 6172792e| ...@....estuary. 00000120 -|6465762f 666f6f12 0b6c6162 656c2d76| dev/foo..label-v 00000130 -|616c7565 0a1e0a0f 65737475 6172792e| alue....estuary. 00000140 -|6465762f 62617212 0b6f7468 65722d76| dev/bar..other-v 00000150 -|616c7565 224208e9 ec061003 1a127333| alue"B........s3 00000160 -|3a2f2f62 75636b65 742f7072 65666978| ://bucket/prefix 00000170 -|220308ac 02320708 3e1080e5 9a773a18| "....2..>....w:. 00000180 -|50617468 7b7b506f 73746669 782e5465| Path{{Postfix.Te 00000190 -|6d706c61 74657d7d 300438cf b0f5015a| mplate}}0.8....Z 000001a0 -|117b2272 65616422 3a227363 68656d61| .{"read":"schema 000001b0 -|227d62bd 07080112 1c7b2264 65726976| "}b......{"deriv 000001c0 -|6174696f 6e223a7b 22636f6e 66696722| ation":{"config" 000001d0 -|3a34327d 7d1ab005 0a0e7472 616e7366| :42}}.....transf 000001e0 -|6f726d5f 6e616d65 12c2030a 1161636d| orm_name.....acm 000001f0 -|65436f2f 636f6c6c 65637469 6f6e1a08| eCo/collection.. 00000200 -|2f6b6579 2f6f6e65 1a082f6b 65792f74| /key/one../key/t 00000210 -|776f220b 2f5f6d65 74612f75 7569642a| wo"./_meta/uuid* 00000220 -|04747970 652a0672 6567696f 6e32a001| .type*.region2.. 00000230 -|0a092f6a 736f6e2f 70747212 07612d66| ../json/ptr..a-f 00000240 -|69656c64 20013287 010a0769 6e746567| ield .2....integ 00000250 -|65720a06 73747269 6e671a13 1a037479| er..string....ty 00000260 -|70220464 61746530 b9603a03 656e6322| p".date0.`:.enc" 00000270 -|05746974 6c652a04 64657363 320e7b22| .title*.desc2.{" 00000280 -|64656622 3a226175 6c74227d 40014a0b| def":"ault"}@.J. 00000290 -|08011100 00000000 408fc052 15080a10| ........@..R.... 000002a0 -|01181422 046e756c 6c220769 6e746567| ...".null".integ 000002b0 -|65725a0b 22636f6e 73742d6f 6e65225a| erZ."const-one"Z 000002c0 -|0b22636f 6e73742d 74776f22 60066803| ."const-two"`.h. 000002d0 -|3a0e7b22 61636b22 3a227472 7565227d| :.{"ack":"true"} 000002e0 -|42127b22 77726974 65223a22 73636865| B.{"write":"sche 000002f0 -|6d61227d 4aa3010a 12706172 74697469| ma"}J....partiti 00000300 -|6f6e2f74 656d706c 61746510 031a400a| on/template...@. 00000310 -|1e0a0f65 73747561 72792e64 65762f66| ...estuary.dev/f 00000320 -|6f6f120b 6c616265 6c2d7661 6c75650a| oo..label-value. 00000330 -|1e0a0f65 73747561 72792e64 65762f62| ...estuary.dev/b 00000340 -|6172120b 6f746865 722d7661 6c756522| ar..other-value" 00000350 -|4208e9ec 0610031a 1273333a 2f2f6275| B........s3://bu 00000360 -|636b6574 2f707265 66697822 0308ac02| cket/prefix".... 00000370 -|3207083e 1080e59a 773a1850 6174687b| 2..>....w:.Path{ 00000380 -|7b506f73 74666978 2e54656d 706c6174| {Postfix.Templat 00000390 -|657d7d30 0438cfb0 f5015a11 7b227265| e}}0.8....Z.{"re 000003a0 -|6164223a 22736368 656d6122 7d1a5b0a| ad":"schema"}.[. 000003b0 -|400a1e0a 0f657374 75617279 2e646576| @....estuary.dev 000003c0 -|2f666f6f 120b6c61 62656c2d 76616c75| /foo..label-valu 000003d0 -|650a1e0a 0f657374 75617279 2e646576| e....estuary.dev 000003e0 -|2f626172 120b6f74 6865722d 76616c75| /bar..other-valu 000003f0 -|6512170a 150a086d 792d6c61 62656c12| e......my-label. 00000400 -|07707265 6669782f 18012001 280e3208| .prefix/.. .(.2. 00000410 -|2f736875 66666c65 32042f6b 65793a18| /shuffle2./key:. 00000420 -|2253454c 45435420 24736875 66666c65| "SELECT $shuffle 00000430 -|2c20246b 65793b22 42137b22 6c616d62| , $key;"B.{"lamb 00000440 -|6461223a 22636f6e 66696722 7d480152| da":"config"}H.R 00000450 -|25646572 6976652f 612f636f 6c6c6563| %derive/a/collec 00000460 -|74696f6e 2f747261 6e73666f 726d5f6e| tion/transform_n 00000470 -|616d652e 76325a06 08cbc8d6 a6066206| ame.v2Z.......b. 00000480 -|0880888b a1066802 22020302 2a80010a| ......h."...*... 00000490 -|0e736861 72642f74 656d706c 6174651a| .shard/template. 000004a0 -|0f726563 6f766572 792f7072 65666978| .recovery/prefix 000004b0 -|220b6869 6e742f70 72656669 78280332| ".hint/prefix(.2 000004c0 -|02083c48 0152400a 1e0a0f65 73747561| .... 00000180 +|e59a773a 18506174 687b7b50 6f737466| ..w:.Path{{Postf 00000190 +|69782e54 656d706c 6174657d 7d300438| ix.Template}}0.8 000001a0 +|cfb0f501 5a117b22 72656164 223a2273| ....Z.{"read":"s 000001b0 +|6368656d 61227d62 c2070801 121c7b22| chema"}b......{" 000001c0 +|64657269 76617469 6f6e223a 7b22636f| derivation":{"co 000001d0 +|6e666967 223a3432 7d7d1ab5 050a0e74| nfig":42}}.....t 000001e0 +|72616e73 666f726d 5f6e616d 6512c703| ransform_name... 000001f0 +|0a116163 6d65436f 2f636f6c 6c656374| ..acmeCo/collect 00000200 +|696f6e1a 082f6b65 792f6f6e 651a082f| ion../key/one../ 00000210 +|6b65792f 74776f22 0b2f5f6d 6574612f| key/two"./_meta/ 00000220 +|75756964 2a047479 70652a06 72656769| uuid*.type*.regi 00000230 +|6f6e32a5 010a092f 6a736f6e 2f707472| on2..../json/ptr 00000240 +|1207612d 6669656c 64200132 8c010a07| ..a-field .2.... 00000250 +|696e7465 6765720a 06737472 696e671a| integer..string. 00000260 +|131a0374 79702204 64617465 30b9603a| ...typ".date0.`: 00000270 +|03656e63 22057469 746c652a 04646573| .enc".title*.des 00000280 +|63320e7b 22646566 223a2261 756c7422| c2.{"def":"ault" 00000290 +|7d40014a 0b080111 00000000 00408fc0| }@.J.........@.. 000002a0 +|5215080a 10011814 22046e75 6c6c2207| R.......".null". 000002b0 +|696e7465 6765725a 0b22636f 6e73742d| integerZ."const- 000002c0 +|6f6e6522 5a0b2263 6f6e7374 2d74776f| one"Z."const-two 000002d0 +|22600668 03720374 79703a0e 7b226163| "`.h.r.typ:.{"ac 000002e0 +|6b223a22 74727565 227d4212 7b227772| k":"true"}B.{"wr 000002f0 +|69746522 3a227363 68656d61 227d4aa3| ite":"schema"}J. 00000300 +|010a1270 61727469 74696f6e 2f74656d| ...partition/tem 00000310 +|706c6174 6510031a 400a1e0a 0f657374| plate...@....est 00000320 +|75617279 2e646576 2f666f6f 120b6c61| uary.dev/foo..la 00000330 +|62656c2d 76616c75 650a1e0a 0f657374| bel-value....est 00000340 +|75617279 2e646576 2f626172 120b6f74| uary.dev/bar..ot 00000350 +|6865722d 76616c75 65224208 e9ec0610| her-value"B..... 00000360 +|031a1273 333a2f2f 6275636b 65742f70| ...s3://bucket/p 00000370 +|72656669 78220308 ac023207 083e1080| refix"....2..>.. 00000380 +|e59a773a 18506174 687b7b50 6f737466| ..w:.Path{{Postf 00000390 +|69782e54 656d706c 6174657d 7d300438| ix.Template}}0.8 000003a0 +|cfb0f501 5a117b22 72656164 223a2273| ....Z.{"read":"s 000003b0 +|6368656d 61227d1a 5b0a400a 1e0a0f65| chema"}.[.@....e 000003c0 +|73747561 72792e64 65762f66 6f6f120b| stuary.dev/foo.. 000003d0 +|6c616265 6c2d7661 6c75650a 1e0a0f65| label-value....e 000003e0 +|73747561 72792e64 65762f62 6172120b| stuary.dev/bar.. 000003f0 +|6f746865 722d7661 6c756512 170a150a| other-value..... 00000400 +|086d792d 6c616265 6c120770 72656669| .my-label..prefi 00000410 +|782f1801 2001280e 32082f73 68756666| x/.. .(.2./shuff 00000420 +|6c653204 2f6b6579 3a182253 454c4543| le2./key:."SELEC 00000430 +|54202473 68756666 6c652c20 246b6579| T $shuffle, $key 00000440 +|3b224213 7b226c61 6d626461 223a2263| ;"B.{"lambda":"c 00000450 +|6f6e6669 67227d48 01522564 65726976| onfig"}H.R%deriv 00000460 +|652f612f 636f6c6c 65637469 6f6e2f74| e/a/collection/t 00000470 +|72616e73 666f726d 5f6e616d 652e7632| ransform_name.v2 00000480 +|5a0608cb c8d6a606 62060880 888ba106| Z.......b....... 00000490 +|68022202 03022a80 010a0e73 68617264| h."...*....shard 000004a0 +|2f74656d 706c6174 651a0f72 65636f76| /template..recov 000004b0 +|6572792f 70726566 6978220b 68696e74| ery/prefix".hint 000004c0 +|2f707265 66697828 03320208 3c480152| /prefix(.2... 000001b0 -|80e59a77 3a185061 74687b7b 506f7374| ...w:.Path{{Post 000001c0 -|6669782e 54656d70 6c617465 7d7d3004| fix.Template}}0. 000001d0 -|38cfb0f5 015a117b 22726561 64223a22| 8....Z.{"read":" 000001e0 -|73636865 6d61227d 22ff030a 0b737461| schema"}"....sta 000001f0 -|626c655f 6e616d65 12c2030a 1161636d| ble_name.....acm 00000200 -|65436f2f 636f6c6c 65637469 6f6e1a08| eCo/collection.. 00000210 -|2f6b6579 2f6f6e65 1a082f6b 65792f74| /key/one../key/t 00000220 -|776f220b 2f5f6d65 74612f75 7569642a| wo"./_meta/uuid* 00000230 -|04747970 652a0672 6567696f 6e32a001| .type*.region2.. 00000240 -|0a092f6a 736f6e2f 70747212 07612d66| ../json/ptr..a-f 00000250 -|69656c64 20013287 010a0769 6e746567| ield .2....integ 00000260 -|65720a06 73747269 6e671a13 1a037479| er..string....ty 00000270 -|70220464 61746530 b9603a03 656e6322| p".date0.`:.enc" 00000280 -|05746974 6c652a04 64657363 320e7b22| .title*.desc2.{" 00000290 -|64656622 3a226175 6c74227d 40014a0b| def":"ault"}@.J. 000002a0 -|08011100 00000000 408fc052 15080a10| ........@..R.... 000002b0 -|01181422 046e756c 6c220769 6e746567| ...".null".integ 000002c0 -|65725a0b 22636f6e 73742d6f 6e65225a| erZ."const-one"Z 000002d0 -|0b22636f 6e73742d 74776f22 60066803| ."const-two"`.h. 000002e0 -|3a0e7b22 61636b22 3a227472 7565227d| :.{"ack":"true"} 000002f0 -|42127b22 77726974 65223a22 73636865| B.{"write":"sche 00000300 -|6d61227d 4aa3010a 12706172 74697469| ma"}J....partiti 00000310 -|6f6e2f74 656d706c 61746510 031a400a| on/template...@. 00000320 -|1e0a0f65 73747561 72792e64 65762f66| ...estuary.dev/f 00000330 -|6f6f120b 6c616265 6c2d7661 6c75650a| oo..label-value. 00000340 -|1e0a0f65 73747561 72792e64 65762f62| ...estuary.dev/b 00000350 -|6172120b 6f746865 722d7661 6c756522| ar..other-value" 00000360 -|4208e9ec 0610031a 1273333a 2f2f6275| B........s3://bu 00000370 -|636b6574 2f707265 66697822 0308ac02| cket/prefix".... 00000380 -|3207083e 1080e59a 773a1850 6174687b| 2..>....w:.Path{ 00000390 -|7b506f73 74666978 2e54656d 706c6174| {Postfix.Templat 000003a0 -|657d7d30 0438cfb0 f5015a11 7b227265| e}}0.8....Z.{"re 000003b0 -|6164223a 22736368 656d6122 7d1a147b| ad":"schema"}..{ 000003c0 -|22736875 66666c65 223a2263 6f6e6669| "shuffle":"confi 000003d0 -|67227d22 137b226c 616d6264 61223a22| g"}".{"lambda":" 000003e0 -|636f6e66 6967227d 28022a02 01023214| config"}(.*...2. 000003f0 -|66696c65 3a2f2f2f 70726f6a 6563742f| file:///project/ 00000400 -|726f6f74 3a320a18 2f757369 6e672f74| root:2../using/t 00000410 -|79706573 63726970 742f6d6f 64756c65| ypescript/module 00000420 -|12166669 6c653a2f 2f2f7061 74682f74| ..file:///path/t 00000430 -|6f2f696d 706f7274 4a0b3030 3a31313a| o/importJ.00:11: 00000440 -|32323a33 331a8404 0ac2030a 1161636d| 22:33........acm 00000450 -|65436f2f 636f6c6c 65637469 6f6e1a08| eCo/collection.. 00000460 -|2f6b6579 2f6f6e65 1a082f6b 65792f74| /key/one../key/t 00000470 -|776f220b 2f5f6d65 74612f75 7569642a| wo"./_meta/uuid* 00000480 -|04747970 652a0672 6567696f 6e32a001| .type*.region2.. 00000490 -|0a092f6a 736f6e2f 70747212 07612d66| ../json/ptr..a-f 000004a0 -|69656c64 20013287 010a0769 6e746567| ield .2....integ 000004b0 -|65720a06 73747269 6e671a13 1a037479| er..string....ty 000004c0 -|70220464 61746530 b9603a03 656e6322| p".date0.`:.enc" 000004d0 -|05746974 6c652a04 64657363 320e7b22| .title*.desc2.{" 000004e0 -|64656622 3a226175 6c74227d 40014a0b| def":"ault"}@.J. 000004f0 -|08011100 00000000 408fc052 15080a10| ........@..R.... 00000500 -|01181422 046e756c 6c220769 6e746567| ...".null".integ 00000510 -|65725a0b 22636f6e 73742d6f 6e65225a| erZ."const-one"Z 00000520 -|0b22636f 6e73742d 74776f22 60066803| ."const-two"`.h. 00000530 -|3a0e7b22 61636b22 3a227472 7565227d| :.{"ack":"true"} 00000540 -|42127b22 77726974 65223a22 73636865| B.{"write":"sche 00000550 -|6d61227d 4aa3010a 12706172 74697469| ma"}J....partiti 00000560 -|6f6e2f74 656d706c 61746510 031a400a| on/template...@. 00000570 -|1e0a0f65 73747561 72792e64 65762f66| ...estuary.dev/f 00000580 -|6f6f120b 6c616265 6c2d7661 6c75650a| oo..label-value. 00000590 -|1e0a0f65 73747561 72792e64 65762f62| ...estuary.dev/b 000005a0 -|6172120b 6f746865 722d7661 6c756522| ar..other-value" 000005b0 -|4208e9ec 0610031a 1273333a 2f2f6275| B........s3://bu 000005c0 -|636b6574 2f707265 66697822 0308ac02| cket/prefix".... 000005d0 -|3207083e 1080e59a 773a1850 6174687b| 2..>....w:.Path{ 000005e0 -|7b506f73 74666978 2e54656d 706c6174| {Postfix.Templat 000005f0 -|657d7d30 0438cfb0 f5015a11 7b227265| e}}0.8....Z.{"re 00000600 -|6164223a 22736368 656d6122 7d120b31| ad":"schema"}..1 00000610 -|313a3232 3a33333a 34341a14 15332211| 1:22:33:44...3". 00000620 -|001d7766 554425bb aa99882d ffeeddcc| ..wfUD%....-.... 00000630 -|2a1a7b22 636f6e6e 6563746f 72223a7b| *.{"connector":{ 00000640 -|22737461 7465223a 34327d7d 223e0802| "state":42}}">.. 00000650 -|121209d2 04000000 00000011 2e160000| ................ 00000660 -|00000000 1a160a09 5b747275 652c3332| ........[true,32 00000670 -|5d120456 4b1e0918 85c39f15 220e7b22| ]..VK.......".{" 00000680 -|72656164 223a2264 6f63227d 2a003266| read":"doc"}*.2f 00000690 -|0a640a4a 0a15612f 72656164 2f6a6f75| .d.J..a/read/jou 000006a0 -|726e616c 3b737566 66697812 3108b960| rnal;suffix.1..` 000006b0 -|12150a05 03090805 07120c09 e3210000| .............!.. 000006c0 -|00000000 10d70812 150a0507 0c662b1d| .............f+. 000006d0 -|120c0935 01000000 00000010 ae111216| ...5............ 000006e0 -|0a0e616e 2f61636b 2f6a6f75 726e616c| ..an/ack/journal 000006f0 -|12040304 02053a00 a2060612 02486918| ......:......Hi. 00000700 -|01| . 00000710 - 00000711 +|73742d74 776f2260 06680372 03747970| st-two"`.h.r.typ 00000110 +|3a0e7b22 61636b22 3a227472 7565227d| :.{"ack":"true"} 00000120 +|42127b22 77726974 65223a22 73636865| B.{"write":"sche 00000130 +|6d61227d 4aa3010a 12706172 74697469| ma"}J....partiti 00000140 +|6f6e2f74 656d706c 61746510 031a400a| on/template...@. 00000150 +|1e0a0f65 73747561 72792e64 65762f66| ...estuary.dev/f 00000160 +|6f6f120b 6c616265 6c2d7661 6c75650a| oo..label-value. 00000170 +|1e0a0f65 73747561 72792e64 65762f62| ...estuary.dev/b 00000180 +|6172120b 6f746865 722d7661 6c756522| ar..other-value" 00000190 +|4208e9ec 0610031a 1273333a 2f2f6275| B........s3://bu 000001a0 +|636b6574 2f707265 66697822 0308ac02| cket/prefix".... 000001b0 +|3207083e 1080e59a 773a1850 6174687b| 2..>....w:.Path{ 000001c0 +|7b506f73 74666978 2e54656d 706c6174| {Postfix.Templat 000001d0 +|657d7d30 0438cfb0 f5015a11 7b227265| e}}0.8....Z.{"re 000001e0 +|6164223a 22736368 656d6122 7d228404| ad":"schema"}".. 000001f0 +|0a0b7374 61626c65 5f6e616d 6512c703| ..stable_name... 00000200 +|0a116163 6d65436f 2f636f6c 6c656374| ..acmeCo/collect 00000210 +|696f6e1a 082f6b65 792f6f6e 651a082f| ion../key/one../ 00000220 +|6b65792f 74776f22 0b2f5f6d 6574612f| key/two"./_meta/ 00000230 +|75756964 2a047479 70652a06 72656769| uuid*.type*.regi 00000240 +|6f6e32a5 010a092f 6a736f6e 2f707472| on2..../json/ptr 00000250 +|1207612d 6669656c 64200132 8c010a07| ..a-field .2.... 00000260 +|696e7465 6765720a 06737472 696e671a| integer..string. 00000270 +|131a0374 79702204 64617465 30b9603a| ...typ".date0.`: 00000280 +|03656e63 22057469 746c652a 04646573| .enc".title*.des 00000290 +|63320e7b 22646566 223a2261 756c7422| c2.{"def":"ault" 000002a0 +|7d40014a 0b080111 00000000 00408fc0| }@.J.........@.. 000002b0 +|5215080a 10011814 22046e75 6c6c2207| R.......".null". 000002c0 +|696e7465 6765725a 0b22636f 6e73742d| integerZ."const- 000002d0 +|6f6e6522 5a0b2263 6f6e7374 2d74776f| one"Z."const-two 000002e0 +|22600668 03720374 79703a0e 7b226163| "`.h.r.typ:.{"ac 000002f0 +|6b223a22 74727565 227d4212 7b227772| k":"true"}B.{"wr 00000300 +|69746522 3a227363 68656d61 227d4aa3| ite":"schema"}J. 00000310 +|010a1270 61727469 74696f6e 2f74656d| ...partition/tem 00000320 +|706c6174 6510031a 400a1e0a 0f657374| plate...@....est 00000330 +|75617279 2e646576 2f666f6f 120b6c61| uary.dev/foo..la 00000340 +|62656c2d 76616c75 650a1e0a 0f657374| bel-value....est 00000350 +|75617279 2e646576 2f626172 120b6f74| uary.dev/bar..ot 00000360 +|6865722d 76616c75 65224208 e9ec0610| her-value"B..... 00000370 +|031a1273 333a2f2f 6275636b 65742f70| ...s3://bucket/p 00000380 +|72656669 78220308 ac023207 083e1080| refix"....2..>.. 00000390 +|e59a773a 18506174 687b7b50 6f737466| ..w:.Path{{Postf 000003a0 +|69782e54 656d706c 6174657d 7d300438| ix.Template}}0.8 000003b0 +|cfb0f501 5a117b22 72656164 223a2273| ....Z.{"read":"s 000003c0 +|6368656d 61227d1a 147b2273 68756666| chema"}..{"shuff 000003d0 +|6c65223a 22636f6e 66696722 7d22137b| le":"config"}".{ 000003e0 +|226c616d 62646122 3a22636f 6e666967| "lambda":"config 000003f0 +|227d2802 2a020102 32146669 6c653a2f| "}(.*...2.file:/ 00000400 +|2f2f7072 6f6a6563 742f726f 6f743a32| //project/root:2 00000410 +|0a182f75 73696e67 2f747970 65736372| ../using/typescr 00000420 +|6970742f 6d6f6475 6c651216 66696c65| ipt/module..file 00000430 +|3a2f2f2f 70617468 2f746f2f 696d706f| :///path/to/impo 00000440 +|72744a0b 30303a31 313a3232 3a33331a| rtJ.00:11:22:33. 00000450 +|89040ac7 030a1161 636d6543 6f2f636f| .......acmeCo/co 00000460 +|6c6c6563 74696f6e 1a082f6b 65792f6f| llection../key/o 00000470 +|6e651a08 2f6b6579 2f74776f 220b2f5f| ne../key/two"./_ 00000480 +|6d657461 2f757569 642a0474 7970652a| meta/uuid*.type* 00000490 +|06726567 696f6e32 a5010a09 2f6a736f| .region2..../jso 000004a0 +|6e2f7074 72120761 2d666965 6c642001| n/ptr..a-field . 000004b0 +|328c010a 07696e74 65676572 0a067374| 2....integer..st 000004c0 +|72696e67 1a131a03 74797022 04646174| ring....typ".dat 000004d0 +|6530b960 3a03656e 63220574 69746c65| e0.`:.enc".title 000004e0 +|2a046465 7363320e 7b226465 66223a22| *.desc2.{"def":" 000004f0 +|61756c74 227d4001 4a0b0801 11000000| ault"}@.J....... 00000500 +|0000408f c0521508 0a100118 1422046e| ..@..R.......".n 00000510 +|756c6c22 07696e74 65676572 5a0b2263| ull".integerZ."c 00000520 +|6f6e7374 2d6f6e65 225a0b22 636f6e73| onst-one"Z."cons 00000530 +|742d7477 6f226006 68037203 7479703a| t-two"`.h.r.typ: 00000540 +|0e7b2261 636b223a 22747275 65227d42| .{"ack":"true"}B 00000550 +|127b2277 72697465 223a2273 6368656d| .{"write":"schem 00000560 +|61227d4a a3010a12 70617274 6974696f| a"}J....partitio 00000570 +|6e2f7465 6d706c61 74651003 1a400a1e| n/template...@.. 00000580 +|0a0f6573 74756172 792e6465 762f666f| ..estuary.dev/fo 00000590 +|6f120b6c 6162656c 2d76616c 75650a1e| o..label-value.. 000005a0 +|0a0f6573 74756172 792e6465 762f6261| ..estuary.dev/ba 000005b0 +|72120b6f 74686572 2d76616c 75652242| r..other-value"B 000005c0 +|08e9ec06 10031a12 73333a2f 2f627563| ........s3://buc 000005d0 +|6b65742f 70726566 69782203 08ac0232| ket/prefix"....2 000005e0 +|07083e10 80e59a77 3a185061 74687b7b| ..>....w:.Path{{ 000005f0 +|506f7374 6669782e 54656d70 6c617465| Postfix.Template 00000600 +|7d7d3004 38cfb0f5 015a117b 22726561| }}0.8....Z.{"rea 00000610 +|64223a22 73636865 6d61227d 120b3131| d":"schema"}..11 00000620 +|3a32323a 33333a34 341a1415 33221100| :22:33:44...3".. 00000630 +|1d776655 4425bbaa 99882dff eeddcc2a| .wfUD%....-....* 00000640 +|1a7b2263 6f6e6e65 63746f72 223a7b22| .{"connector":{" 00000650 +|73746174 65223a34 327d7d22 3e080212| state":42}}">... 00000660 +|1209d204 00000000 0000112e 16000000| ................ 00000670 +|0000001a 160a095b 74727565 2c33325d| .......[true,32] 00000680 +|1204564b 1e091885 c39f1522 0e7b2272| ..VK.......".{"r 00000690 +|65616422 3a22646f 63227d2a 0032660a| ead":"doc"}*.2f. 000006a0 +|640a4a0a 15612f72 6561642f 6a6f7572| d.J..a/read/jour 000006b0 +|6e616c3b 73756666 69781231 08b96012| nal;suffix.1..`. 000006c0 +|150a0503 09080507 120c09e3 21000000| ............!... 000006d0 +|00000010 d7081215 0a05070c 662b1d12| ............f+.. 000006e0 +|0c093501 00000000 000010ae 1112160a| ..5............. 000006f0 +|0e616e2f 61636b2f 6a6f7572 6e616c12| .an/ack/journal. 00000700 +|04030402 053a00a2 06061202 48691801| .....:......Hi.. 00000710 + 00000720 diff --git a/crates/proto-flow/tests/snapshots/regression__materialization_spec_json.snap b/crates/proto-flow/tests/snapshots/regression__materialization_spec_json.snap index 17ed650a5d3..2cbf9bf0b0c 100644 --- a/crates/proto-flow/tests/snapshots/regression__materialization_spec_json.snap +++ b/crates/proto-flow/tests/snapshots/regression__materialization_spec_json.snap @@ -1,6 +1,5 @@ --- source: crates/proto-flow/tests/regression.rs -assertion_line: 932 expression: json_test(msg) --- { @@ -65,7 +64,8 @@ expression: json_test(msg) "const-two" ], "reduce": "REDUCE_MERGE", - "redact": "REDACT_SHA256" + "redact": "REDACT_SHA256", + "contentMediaType": "typ" } } ], diff --git a/crates/proto-flow/tests/snapshots/regression__materialization_spec_proto.snap b/crates/proto-flow/tests/snapshots/regression__materialization_spec_proto.snap index f580275ca3b..2c1f0001469 100644 --- a/crates/proto-flow/tests/snapshots/regression__materialization_spec_proto.snap +++ b/crates/proto-flow/tests/snapshots/regression__materialization_spec_proto.snap @@ -1,20 +1,19 @@ --- source: crates/proto-flow/tests/regression.rs -assertion_line: 938 expression: proto_test(msg) --- |0a166163 6d65436f 2f6d6174 65726961| ..acmeCo/materia 00000000 |6c697a61 74696f6e 10081a1d 7b226d61| lization....{"ma 00000010 |74657269 616c697a 65223a7b 22636f6e| terialize":{"con 00000020 -|66696722 3a34327d 7d22ff05 0a157b22| fig":42}}"....{" 00000030 +|66696722 3a34327d 7d228406 0a157b22| fig":42}}"....{" 00000030 |7265736f 75726365 223a2263 6f6e6669| resource":"confi 00000040 |67227d12 04736f6d 65120470 6174681a| g"}..some..path. 00000050 -|c2030a11 61636d65 436f2f63 6f6c6c65| ....acmeCo/colle 00000060 +|c7030a11 61636d65 436f2f63 6f6c6c65| ....acmeCo/colle 00000060 |6374696f 6e1a082f 6b65792f 6f6e651a| ction../key/one. 00000070 |082f6b65 792f7477 6f220b2f 5f6d6574| ./key/two"./_met 00000080 |612f7575 69642a04 74797065 2a067265| a/uuid*.type*.re 00000090 -|67696f6e 32a0010a 092f6a73 6f6e2f70| gion2..../json/p 000000a0 -|74721207 612d6669 656c6420 01328701| tr..a-field .2.. 000000b0 +|67696f6e 32a5010a 092f6a73 6f6e2f70| gion2..../json/p 000000a0 +|74721207 612d6669 656c6420 01328c01| tr..a-field .2.. 000000b0 |0a07696e 74656765 720a0673 7472696e| ..integer..strin 000000c0 |671a131a 03747970 22046461 746530b9| g....typ".date0. 000000d0 |603a0365 6e632205 7469746c 652a0464| `:.enc".title*.d 000000e0 @@ -23,54 +22,54 @@ expression: proto_test(msg) |8fc05215 080a1001 18142204 6e756c6c| ..R.......".null 00000110 |2207696e 74656765 725a0b22 636f6e73| ".integerZ."cons 00000120 |742d6f6e 65225a0b 22636f6e 73742d74| t-one"Z."const-t 00000130 -|776f2260 0668033a 0e7b2261 636b223a| wo"`.h.:.{"ack": 00000140 -|22747275 65227d42 127b2277 72697465| "true"}B.{"write 00000150 -|223a2273 6368656d 61227d4a a3010a12| ":"schema"}J.... 00000160 -|70617274 6974696f 6e2f7465 6d706c61| partition/templa 00000170 -|74651003 1a400a1e 0a0f6573 74756172| te...@....estuar 00000180 -|792e6465 762f666f 6f120b6c 6162656c| y.dev/foo..label 00000190 -|2d76616c 75650a1e 0a0f6573 74756172| -value....estuar 000001a0 -|792e6465 762f6261 72120b6f 74686572| y.dev/bar..other 000001b0 -|2d76616c 75652242 08e9ec06 10031a12| -value"B........ 000001c0 -|73333a2f 2f627563 6b65742f 70726566| s3://bucket/pref 000001d0 -|69782203 08ac0232 07083e10 80e59a77| ix"....2..>....w 000001e0 -|3a185061 74687b7b 506f7374 6669782e| :.Path{{Postfix. 000001f0 -|54656d70 6c617465 7d7d3004 38cfb0f5| Template}}0.8... 00000200 -|015a117b 22726561 64223a22 73636865| .Z.{"read":"sche 00000210 -|6d61227d 22550a07 6b65792f 6f6e6512| ma"}"U..key/one. 00000220 -|0776616c 2f74776f 1a0d666c 6f775f64| .val/two..flow_d 00000230 -|6f63756d 656e7422 1d0a0761 5f666965| ocument"...a_fie 00000240 -|6c641212 7b226669 656c6422 3a22636f| ld..{"field":"co 00000250 -|6e666967 227d2213 0a0b6f74 6865722f| nfig"}"...other/ 00000260 -|6669656c 64120434 322e353a 5b0a400a| field..42.5:[.@. 00000270 -|1e0a0f65 73747561 72792e64 65762f66| ...estuary.dev/f 00000280 -|6f6f120b 6c616265 6c2d7661 6c75650a| oo..label-value. 00000290 -|1e0a0f65 73747561 72792e64 65762f62| ...estuary.dev/b 000002a0 -|6172120b 6f746865 722d7661 6c756512| ar..other-value. 000002b0 -|170a150a 086d792d 6c616265 6c120770| .....my-label..p 000002c0 -|72656669 782f1801 42316d61 74657269| refix/..B1materi 000002d0 -|616c697a 652f6163 6d65436f 2f6d6174| alize/acmeCo/mat 000002e0 -|65726961 6c697a61 74696f6e 2f736f6d| erialization/som 000002f0 -|65253230 70617468 2e763148 03520608| e%20path.v1H.R.. 00000300 -|cbc8d6a6 065a0608 80888ba1 0660016a| .....Z.......`.j 00000310 -|0e736f6d 65253230 70617468 2e763172| .some%20path.v1r 00000320 -|0a088080 0410e807 18e8072a 80010a0e| ...........*.... 00000330 -|73686172 642f7465 6d706c61 74651a0f| shard/template.. 00000340 -|7265636f 76657279 2f707265 66697822| recovery/prefix" 00000350 -|0b68696e 742f7072 65666978 28033202| .hint/prefix(.2. 00000360 -|083c4801 52400a1e 0a0f6573 74756172| . 000001e0 +|1080e59a 773a1850 6174687b 7b506f73| ....w:.Path{{Pos 000001f0 +|74666978 2e54656d 706c6174 657d7d30| tfix.Template}}0 00000200 +|0438cfb0 f5015a11 7b227265 6164223a| .8....Z.{"read": 00000210 +|22736368 656d6122 7d22550a 076b6579| "schema"}"U..key 00000220 +|2f6f6e65 12077661 6c2f7477 6f1a0d66| /one..val/two..f 00000230 +|6c6f775f 646f6375 6d656e74 221d0a07| low_document"... 00000240 +|615f6669 656c6412 127b2266 69656c64| a_field..{"field 00000250 +|223a2263 6f6e6669 67227d22 130a0b6f| ":"config"}"...o 00000260 +|74686572 2f666965 6c641204 34322e35| ther/field..42.5 00000270 +|3a5b0a40 0a1e0a0f 65737475 6172792e| :[.@....estuary. 00000280 +|6465762f 666f6f12 0b6c6162 656c2d76| dev/foo..label-v 00000290 +|616c7565 0a1e0a0f 65737475 6172792e| alue....estuary. 000002a0 +|6465762f 62617212 0b6f7468 65722d76| dev/bar..other-v 000002b0 +|616c7565 12170a15 0a086d79 2d6c6162| alue......my-lab 000002c0 +|656c1207 70726566 69782f18 0142316d| el..prefix/..B1m 000002d0 +|61746572 69616c69 7a652f61 636d6543| aterialize/acmeC 000002e0 +|6f2f6d61 74657269 616c697a 6174696f| o/materializatio 000002f0 +|6e2f736f 6d652532 30706174 682e7631| n/some%20path.v1 00000300 +|48035206 08cbc8d6 a6065a06 0880888b| H.R.......Z..... 00000310 +|a1066001 6a0e736f 6d652532 30706174| ..`.j.some%20pat 00000320 +|682e7631 720a0880 800410e8 0718e807| h.v1r........... 00000330 +|2a80010a 0e736861 72642f74 656d706c| *....shard/templ 00000340 +|6174651a 0f726563 6f766572 792f7072| ate..recovery/pr 00000350 +|65666978 220b6869 6e742f70 72656669| efix".hint/prefi 00000360 +|78280332 02083c48 0152400a 1e0a0f65| x(.2......w:.Path{{ 000001f0 -|506f7374 6669782e 54656d70 6c617465| Postfix.Template 00000200 -|7d7d3004 38cfb0f5 015a117b 22726561| }}0.8....Z.{"rea 00000210 -|64223a22 73636865 6d61227d 1a1d0a07| d":"schema"}.... 00000220 -|615f6669 656c6412 127b2266 69656c64| a_field..{"field 00000230 -|223a2263 6f6e6669 67227d1a 130a0b6f| ":"config"}....o 00000240 -|74686572 2f666965 6c641204 34322e35| ther/field..42.5 00000250 -|20032a07 6b65792f 6f6e6532 0b30303a| .*.key/one2.00: 00000260 -|31313a32 323a3333 1a8f090a db080a16| 11:22:33........ 00000270 -|61636d65 436f2f6d 61746572 69616c69| acmeCo/materiali 00000280 -|7a617469 6f6e1008 1a1d7b22 6d617465| zation....{"mate 00000290 -|7269616c 697a6522 3a7b2263 6f6e6669| rialize":{"confi 000002a0 -|67223a34 327d7d22 ff050a15 7b227265| g":42}}"....{"re 000002b0 -|736f7572 6365223a 22636f6e 66696722| source":"config" 000002c0 -|7d120473 6f6d6512 04706174 681ac203| }..some..path... 000002d0 -|0a116163 6d65436f 2f636f6c 6c656374| ..acmeCo/collect 000002e0 -|696f6e1a 082f6b65 792f6f6e 651a082f| ion../key/one../ 000002f0 -|6b65792f 74776f22 0b2f5f6d 6574612f| key/two"./_meta/ 00000300 -|75756964 2a047479 70652a06 72656769| uuid*.type*.regi 00000310 -|6f6e32a0 010a092f 6a736f6e 2f707472| on2..../json/ptr 00000320 -|1207612d 6669656c 64200132 87010a07| ..a-field .2.... 00000330 -|696e7465 6765720a 06737472 696e671a| integer..string. 00000340 -|131a0374 79702204 64617465 30b9603a| ...typ".date0.`: 00000350 -|03656e63 22057469 746c652a 04646573| .enc".title*.des 00000360 -|63320e7b 22646566 223a2261 756c7422| c2.{"def":"ault" 00000370 -|7d40014a 0b080111 00000000 00408fc0| }@.J.........@.. 00000380 -|5215080a 10011814 22046e75 6c6c2207| R.......".null". 00000390 -|696e7465 6765725a 0b22636f 6e73742d| integerZ."const- 000003a0 -|6f6e6522 5a0b2263 6f6e7374 2d74776f| one"Z."const-two 000003b0 -|22600668 033a0e7b 2261636b 223a2274| "`.h.:.{"ack":"t 000003c0 -|72756522 7d42127b 22777269 7465223a| rue"}B.{"write": 000003d0 -|22736368 656d6122 7d4aa301 0a127061| "schema"}J....pa 000003e0 -|72746974 696f6e2f 74656d70 6c617465| rtition/template 000003f0 -|10031a40 0a1e0a0f 65737475 6172792e| ...@....estuary. 00000400 -|6465762f 666f6f12 0b6c6162 656c2d76| dev/foo..label-v 00000410 -|616c7565 0a1e0a0f 65737475 6172792e| alue....estuary. 00000420 -|6465762f 62617212 0b6f7468 65722d76| dev/bar..other-v 00000430 -|616c7565 224208e9 ec061003 1a127333| alue"B........s3 00000440 -|3a2f2f62 75636b65 742f7072 65666978| ://bucket/prefix 00000450 -|220308ac 02320708 3e1080e5 9a773a18| "....2..>....w:. 00000460 -|50617468 7b7b506f 73746669 782e5465| Path{{Postfix.Te 00000470 -|6d706c61 74657d7d 300438cf b0f5015a| mplate}}0.8....Z 00000480 -|117b2272 65616422 3a227363 68656d61| .{"read":"schema 00000490 -|227d2255 0a076b65 792f6f6e 65120776| "}"U..key/one..v 000004a0 -|616c2f74 776f1a0d 666c6f77 5f646f63| al/two..flow_doc 000004b0 -|756d656e 74221d0a 07615f66 69656c64| ument"...a_field 000004c0 -|12127b22 6669656c 64223a22 636f6e66| ..{"field":"conf 000004d0 -|6967227d 22130a0b 6f746865 722f6669| ig"}"...other/fi 000004e0 -|656c6412 0434322e 353a5b0a 400a1e0a| eld..42.5:[.@... 000004f0 -|0f657374 75617279 2e646576 2f666f6f| .estuary.dev/foo 00000500 -|120b6c61 62656c2d 76616c75 650a1e0a| ..label-value... 00000510 -|0f657374 75617279 2e646576 2f626172| .estuary.dev/bar 00000520 -|120b6f74 6865722d 76616c75 6512170a| ..other-value... 00000530 -|150a086d 792d6c61 62656c12 07707265| ...my-label..pre 00000540 -|6669782f 18014231 6d617465 7269616c| fix/..B1material 00000550 -|697a652f 61636d65 436f2f6d 61746572| ize/acmeCo/mater 00000560 -|69616c69 7a617469 6f6e2f73 6f6d6525| ialization/some% 00000570 -|32307061 74682e76 31480352 0608cbc8| 20path.v1H.R.... 00000580 -|d6a6065a 06088088 8ba10660 016a0e73| ...Z.......`.j.s 00000590 -|6f6d6525 32307061 74682e76 31720a08| ome%20path.v1r.. 000005a0 -|80800410 e80718e8 072a8001 0a0e7368| .........*....sh 000005b0 -|6172642f 74656d70 6c617465 1a0f7265| ard/template..re 000005c0 -|636f7665 72792f70 72656669 78220b68| covery/prefix".h 000005d0 -|696e742f 70726566 69782803 3202083c| int/prefix(.2..< 000005e0 -|48015240 0a1e0a0f 65737475 6172792e| H.R@....estuary. 000005f0 -|6465762f 666f6f12 0b6c6162 656c2d76| dev/foo..label-v 00000600 -|616c7565 0a1e0a0f 65737475 6172792e| alue....estuary. 00000610 -|6465762f 62617212 0b6f7468 65722d76| dev/bar..other-v 00000620 -|616c7565 608cdc02 68e9ec06 32380a11| alue`...h...28.. 00000630 -|7265636f 76657279 2f74656d 706c6174| recovery/templat 00000640 -|65100322 21088080 4010041a 1473333a| e.."!...@....s3: 00000650 -|2f2f6275 636b6574 2f726563 6f766572| //bucket/recover 00000660 -|79220308 ac023a0c 08903f12 05687474| y"....:...?..htt 00000670 -|70731801 3a0308a8 46524e5b 7b226d65| ps..:...FRN[{"me 00000680 -|74686f64 223a2250 4f535422 2c227061| thod":"POST","pa 00000690 -|796c6f61 6454656d 706c6174 65223a22| yloadTemplate":" 000006a0 -|7b7d222c 2275726c 223a2268 74747073| {}","url":"https 000006b0 -|3a2f2f65 78616d70 6c652e63 6f6d2f77| ://example.com/w 000006c0 -|6562686f 6f6b227d 5d120b31 313a3232| ebhook"}]..11:22 000006d0 -|3a33333a 34342a0b 30303a31 313a3232| :33:44*.00:11:22 000006e0 -|3a333332 157b2263 6f6e6e65 63746f72| :332.{"connector 000006f0 -|223a2273 74617465 227d229d 090adb08| ":"state"}"..... 00000700 -|0a166163 6d65436f 2f6d6174 65726961| ..acmeCo/materia 00000710 -|6c697a61 74696f6e 10081a1d 7b226d61| lization....{"ma 00000720 -|74657269 616c697a 65223a7b 22636f6e| terialize":{"con 00000730 -|66696722 3a34327d 7d22ff05 0a157b22| fig":42}}"....{" 00000740 -|7265736f 75726365 223a2263 6f6e6669| resource":"confi 00000750 -|67227d12 04736f6d 65120470 6174681a| g"}..some..path. 00000760 -|c2030a11 61636d65 436f2f63 6f6c6c65| ....acmeCo/colle 00000770 -|6374696f 6e1a082f 6b65792f 6f6e651a| ction../key/one. 00000780 -|082f6b65 792f7477 6f220b2f 5f6d6574| ./key/two"./_met 00000790 -|612f7575 69642a04 74797065 2a067265| a/uuid*.type*.re 000007a0 -|67696f6e 32a0010a 092f6a73 6f6e2f70| gion2..../json/p 000007b0 -|74721207 612d6669 656c6420 01328701| tr..a-field .2.. 000007c0 -|0a07696e 74656765 720a0673 7472696e| ..integer..strin 000007d0 -|671a131a 03747970 22046461 746530b9| g....typ".date0. 000007e0 -|603a0365 6e632205 7469746c 652a0464| `:.enc".title*.d 000007f0 -|65736332 0e7b2264 6566223a 2261756c| esc2.{"def":"aul 00000800 -|74227d40 014a0b08 01110000 00000040| t"}@.J.........@ 00000810 -|8fc05215 080a1001 18142204 6e756c6c| ..R.......".null 00000820 -|2207696e 74656765 725a0b22 636f6e73| ".integerZ."cons 00000830 -|742d6f6e 65225a0b 22636f6e 73742d74| t-one"Z."const-t 00000840 -|776f2260 0668033a 0e7b2261 636b223a| wo"`.h.:.{"ack": 00000850 -|22747275 65227d42 127b2277 72697465| "true"}B.{"write 00000860 -|223a2273 6368656d 61227d4a a3010a12| ":"schema"}J.... 00000870 -|70617274 6974696f 6e2f7465 6d706c61| partition/templa 00000880 -|74651003 1a400a1e 0a0f6573 74756172| te...@....estuar 00000890 -|792e6465 762f666f 6f120b6c 6162656c| y.dev/foo..label 000008a0 -|2d76616c 75650a1e 0a0f6573 74756172| -value....estuar 000008b0 -|792e6465 762f6261 72120b6f 74686572| y.dev/bar..other 000008c0 -|2d76616c 75652242 08e9ec06 10031a12| -value"B........ 000008d0 -|73333a2f 2f627563 6b65742f 70726566| s3://bucket/pref 000008e0 -|69782203 08ac0232 07083e10 80e59a77| ix"....2..>....w 000008f0 -|3a185061 74687b7b 506f7374 6669782e| :.Path{{Postfix. 00000900 -|54656d70 6c617465 7d7d3004 38cfb0f5| Template}}0.8... 00000910 -|015a117b 22726561 64223a22 73636865| .Z.{"read":"sche 00000920 -|6d61227d 22550a07 6b65792f 6f6e6512| ma"}"U..key/one. 00000930 -|0776616c 2f74776f 1a0d666c 6f775f64| .val/two..flow_d 00000940 -|6f63756d 656e7422 1d0a0761 5f666965| ocument"...a_fie 00000950 -|6c641212 7b226669 656c6422 3a22636f| ld..{"field":"co 00000960 -|6e666967 227d2213 0a0b6f74 6865722f| nfig"}"...other/ 00000970 -|6669656c 64120434 322e353a 5b0a400a| field..42.5:[.@. 00000980 -|1e0a0f65 73747561 72792e64 65762f66| ...estuary.dev/f 00000990 -|6f6f120b 6c616265 6c2d7661 6c75650a| oo..label-value. 000009a0 -|1e0a0f65 73747561 72792e64 65762f62| ...estuary.dev/b 000009b0 -|6172120b 6f746865 722d7661 6c756512| ar..other-value. 000009c0 -|170a150a 086d792d 6c616265 6c120770| .....my-label..p 000009d0 -|72656669 782f1801 42316d61 74657269| refix/..B1materi 000009e0 -|616c697a 652f6163 6d65436f 2f6d6174| alize/acmeCo/mat 000009f0 -|65726961 6c697a61 74696f6e 2f736f6d| erialization/som 00000a00 -|65253230 70617468 2e763148 03520608| e%20path.v1H.R.. 00000a10 -|cbc8d6a6 065a0608 80888ba1 0660016a| .....Z.......`.j 00000a20 -|0e736f6d 65253230 70617468 2e763172| .some%20path.v1r 00000a30 -|0a088080 0410e807 18e8072a 80010a0e| ...........*.... 00000a40 -|73686172 642f7465 6d706c61 74651a0f| shard/template.. 00000a50 -|7265636f 76657279 2f707265 66697822| recovery/prefix" 00000a60 -|0b68696e 742f7072 65666978 28033202| .hint/prefix(.2. 00000a70 -|083c4801 52400a1e 0a0f6573 74756172| .....w:.P 000001f0 +|6174687b 7b506f73 74666978 2e54656d| ath{{Postfix.Tem 00000200 +|706c6174 657d7d30 0438cfb0 f5015a11| plate}}0.8....Z. 00000210 +|7b227265 6164223a 22736368 656d6122| {"read":"schema" 00000220 +|7d1a1d0a 07615f66 69656c64 12127b22| }....a_field..{" 00000230 +|6669656c 64223a22 636f6e66 6967227d| field":"config"} 00000240 +|1a130a0b 6f746865 722f6669 656c6412| ....other/field. 00000250 +|0434322e 3520032a 076b6579 2f6f6e65| .42.5 .*.key/one 00000260 +|320b3030 3a31313a 32323a33 331a9409| 2.00:11:22:33... 00000270 +|0ae0080a 1661636d 65436f2f 6d617465| .....acmeCo/mate 00000280 +|7269616c 697a6174 696f6e10 081a1d7b| rialization....{ 00000290 +|226d6174 65726961 6c697a65 223a7b22| "materialize":{" 000002a0 +|636f6e66 6967223a 34327d7d 2284060a| config":42}}"... 000002b0 +|157b2272 65736f75 72636522 3a22636f| .{"resource":"co 000002c0 +|6e666967 227d1204 736f6d65 12047061| nfig"}..some..pa 000002d0 +|74681ac7 030a1161 636d6543 6f2f636f| th.....acmeCo/co 000002e0 +|6c6c6563 74696f6e 1a082f6b 65792f6f| llection../key/o 000002f0 +|6e651a08 2f6b6579 2f74776f 220b2f5f| ne../key/two"./_ 00000300 +|6d657461 2f757569 642a0474 7970652a| meta/uuid*.type* 00000310 +|06726567 696f6e32 a5010a09 2f6a736f| .region2..../jso 00000320 +|6e2f7074 72120761 2d666965 6c642001| n/ptr..a-field . 00000330 +|328c010a 07696e74 65676572 0a067374| 2....integer..st 00000340 +|72696e67 1a131a03 74797022 04646174| ring....typ".dat 00000350 +|6530b960 3a03656e 63220574 69746c65| e0.`:.enc".title 00000360 +|2a046465 7363320e 7b226465 66223a22| *.desc2.{"def":" 00000370 +|61756c74 227d4001 4a0b0801 11000000| ault"}@.J....... 00000380 +|0000408f c0521508 0a100118 1422046e| ..@..R.......".n 00000390 +|756c6c22 07696e74 65676572 5a0b2263| ull".integerZ."c 000003a0 +|6f6e7374 2d6f6e65 225a0b22 636f6e73| onst-one"Z."cons 000003b0 +|742d7477 6f226006 68037203 7479703a| t-two"`.h.r.typ: 000003c0 +|0e7b2261 636b223a 22747275 65227d42| .{"ack":"true"}B 000003d0 +|127b2277 72697465 223a2273 6368656d| .{"write":"schem 000003e0 +|61227d4a a3010a12 70617274 6974696f| a"}J....partitio 000003f0 +|6e2f7465 6d706c61 74651003 1a400a1e| n/template...@.. 00000400 +|0a0f6573 74756172 792e6465 762f666f| ..estuary.dev/fo 00000410 +|6f120b6c 6162656c 2d76616c 75650a1e| o..label-value.. 00000420 +|0a0f6573 74756172 792e6465 762f6261| ..estuary.dev/ba 00000430 +|72120b6f 74686572 2d76616c 75652242| r..other-value"B 00000440 +|08e9ec06 10031a12 73333a2f 2f627563| ........s3://buc 00000450 +|6b65742f 70726566 69782203 08ac0232| ket/prefix"....2 00000460 +|07083e10 80e59a77 3a185061 74687b7b| ..>....w:.Path{{ 00000470 +|506f7374 6669782e 54656d70 6c617465| Postfix.Template 00000480 +|7d7d3004 38cfb0f5 015a117b 22726561| }}0.8....Z.{"rea 00000490 +|64223a22 73636865 6d61227d 22550a07| d":"schema"}"U.. 000004a0 +|6b65792f 6f6e6512 0776616c 2f74776f| key/one..val/two 000004b0 +|1a0d666c 6f775f64 6f63756d 656e7422| ..flow_document" 000004c0 +|1d0a0761 5f666965 6c641212 7b226669| ...a_field..{"fi 000004d0 +|656c6422 3a22636f 6e666967 227d2213| eld":"config"}". 000004e0 +|0a0b6f74 6865722f 6669656c 64120434| ..other/field..4 000004f0 +|322e353a 5b0a400a 1e0a0f65 73747561| 2.5:[.@....estua 00000500 +|72792e64 65762f66 6f6f120b 6c616265| ry.dev/foo..labe 00000510 +|6c2d7661 6c75650a 1e0a0f65 73747561| l-value....estua 00000520 +|72792e64 65762f62 6172120b 6f746865| ry.dev/bar..othe 00000530 +|722d7661 6c756512 170a150a 086d792d| r-value......my- 00000540 +|6c616265 6c120770 72656669 782f1801| label..prefix/.. 00000550 +|42316d61 74657269 616c697a 652f6163| B1materialize/ac 00000560 +|6d65436f 2f6d6174 65726961 6c697a61| meCo/materializa 00000570 +|74696f6e 2f736f6d 65253230 70617468| tion/some%20path 00000580 +|2e763148 03520608 cbc8d6a6 065a0608| .v1H.R.......Z.. 00000590 +|80888ba1 0660016a 0e736f6d 65253230| .....`.j.some%20 000005a0 +|70617468 2e763172 0a088080 0410e807| path.v1r........ 000005b0 +|18e8072a 80010a0e 73686172 642f7465| ...*....shard/te 000005c0 +|6d706c61 74651a0f 7265636f 76657279| mplate..recovery 000005d0 +|2f707265 66697822 0b68696e 742f7072| /prefix".hint/pr 000005e0 +|65666978 28033202 083c4801 52400a1e| efix(.2......w: 00000900 +|18506174 687b7b50 6f737466 69782e54| .Path{{Postfix.T 00000910 +|656d706c 6174657d 7d300438 cfb0f501| emplate}}0.8.... 00000920 +|5a117b22 72656164 223a2273 6368656d| Z.{"read":"schem 00000930 +|61227d22 550a076b 65792f6f 6e651207| a"}"U..key/one.. 00000940 +|76616c2f 74776f1a 0d666c6f 775f646f| val/two..flow_do 00000950 +|63756d65 6e74221d 0a07615f 6669656c| cument"...a_fiel 00000960 +|6412127b 22666965 6c64223a 22636f6e| d..{"field":"con 00000970 +|66696722 7d22130a 0b6f7468 65722f66| fig"}"...other/f 00000980 +|69656c64 12043432 2e353a5b 0a400a1e| ield..42.5:[.@.. 00000990 +|0a0f6573 74756172 792e6465 762f666f| ..estuary.dev/fo 000009a0 +|6f120b6c 6162656c 2d76616c 75650a1e| o..label-value.. 000009b0 +|0a0f6573 74756172 792e6465 762f6261| ..estuary.dev/ba 000009c0 +|72120b6f 74686572 2d76616c 75651217| r..other-value.. 000009d0 +|0a150a08 6d792d6c 6162656c 12077072| ....my-label..pr 000009e0 +|65666978 2f180142 316d6174 65726961| efix/..B1materia 000009f0 +|6c697a65 2f61636d 65436f2f 6d617465| lize/acmeCo/mate 00000a00 +|7269616c 697a6174 696f6e2f 736f6d65| rialization/some 00000a10 +|25323070 6174682e 76314803 520608cb| %20path.v1H.R... 00000a20 +|c8d6a606 5a060880 888ba106 60016a0e| ....Z.......`.j. 00000a30 +|736f6d65 25323070 6174682e 7631720a| some%20path.v1r. 00000a40 +|08808004 10e80718 e8072a80 010a0e73| ..........*....s 00000a50 +|68617264 2f74656d 706c6174 651a0f72| hard/template..r 00000a60 +|65636f76 6572792f 70726566 6978220b| ecovery/prefix". 00000a70 +|68696e74 2f707265 66697828 03320208| hint/prefix(.2.. 00000a80 +|3c480152 400a1e0a 0f657374 75617279| 0 { + i -= len(m.ContentMediaType) + copy(dAtA[i:], m.ContentMediaType) + i = encodeVarintFlow(dAtA, i, uint64(len(m.ContentMediaType))) + i-- + dAtA[i] = 0x72 + } if m.Redact != 0 { i = encodeVarintFlow(dAtA, i, uint64(m.Redact)) i-- @@ -5527,6 +5543,10 @@ func (m *Inference) ProtoSize() (n int) { if m.Redact != 0 { n += 1 + sovFlow(uint64(m.Redact)) } + l = len(m.ContentMediaType) + if l > 0 { + n += 1 + l + sovFlow(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -7301,6 +7321,38 @@ func (m *Inference) Unmarshal(dAtA []byte) error { break } } + case 14: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ContentMediaType", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFlow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthFlow + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthFlow + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ContentMediaType = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipFlow(dAtA[iNdEx:]) diff --git a/go/protocols/flow/flow.proto b/go/protocols/flow/flow.proto index c251bdf031a..bfb151ba036 100644 --- a/go/protocols/flow/flow.proto +++ b/go/protocols/flow/flow.proto @@ -119,6 +119,7 @@ message Inference { message String { reserved 1, 2, 5; // Annotated Content-Type when the projection is of "string" type. + // Deprecated for non-string projections; prefer `Inference.content_media_type`. string content_type = 3; // Annotated format when the projection is of "string" type. string format = 4; @@ -186,6 +187,12 @@ message Inference { Reduce reduce = 12; // Applied `redact` strategy. Redact redact = 13; + // Annotated `contentMediaType` of the schema. The JSON-Schema spec + // defines this annotation only for strings; Flow extends it to apply + // to any type, so it surfaces at the top level. For backward + // compatibility, `Inference.String.content_type` continues to be populated + // whenever this projection includes the "string" type. + string content_media_type = 14; } message NetworkPort {