diff --git a/README.md b/README.md index 7df44b3..e040dbc 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,17 @@ to the default value for its type and the EOF will be ignored. Specifies a static string of bytes which will be written or has to be present when reading before a given field. +## Struct attributes + +### `#[speedy(non_exhaustive)]` + +The number of field in the structure is written before the struct fields. +This allows you to append fields to a struct without losing the ability to read data written with the old struct. +Warning : this is not retroactive, meaning if a struct did not have this tag before, +you won't be able to read serialized data serialized without this attribute. +This features add a varint64 in front of each serialized structs (1 byte if less than 128 fields). + + ## Enum attributes ### `#[speedy(tag_type = $ty)]` diff --git a/speedy-derive/src/lib.rs b/speedy-derive/src/lib.rs index 0820a9a..8b2bf03 100644 --- a/speedy-derive/src/lib.rs +++ b/speedy-derive/src/lib.rs @@ -47,8 +47,11 @@ mod kw { syn::custom_keyword!( skip ); syn::custom_keyword!( constant_prefix ); syn::custom_keyword!( peek_tag ); + + syn::custom_keyword!( non_exhaustive ); syn::custom_keyword!( varint ); + syn::custom_keyword!( u7 ); syn::custom_keyword!( u8 ); syn::custom_keyword!( u16 ); @@ -388,6 +391,9 @@ enum VariantAttribute { } enum StructAttribute { + NonExhaustive { + key_token: kw::non_exhaustive + } } enum EnumAttribute { @@ -430,11 +436,16 @@ fn parse_variant_attribute( } fn parse_struct_attribute( - _input: &syn::parse::ParseStream, - _lookahead: &syn::parse::Lookahead1 + input: &syn::parse::ParseStream, + lookahead: &syn::parse::Lookahead1 ) -> syn::parse::Result< Option< StructAttribute > > { - Ok( None ) + if lookahead.peek( kw::non_exhaustive ) { + let key_token = input.parse::< kw::non_exhaustive >()?; + Ok( Some( StructAttribute::NonExhaustive { key_token } ) ) + } else { + Ok( None ) + } } fn parse_enum_attribute( @@ -497,6 +508,7 @@ struct VariantAttributes { } struct StructAttributes { + non_exhaustive: bool } struct EnumAttributes { @@ -577,10 +589,21 @@ fn collect_variant_attributes( attrs: Vec< VariantAttribute > ) -> Result< Varia } fn collect_struct_attributes( attrs: Vec< StructAttribute > ) -> Result< StructAttributes, syn::Error > { - for _attr in attrs { + let mut non_exhaustive = false; + for attr in attrs { + match attr { + StructAttribute::NonExhaustive { key_token } => { + if non_exhaustive { + let message = "Duplicate 'non_exhaustive'"; + return Err( syn::Error::new( key_token.span(), message ) ); + } + non_exhaustive = true; + } + } } Ok( StructAttributes { + non_exhaustive }) } @@ -621,33 +644,45 @@ enum StructKind { struct Struct< 'a > { fields: Vec< Field< 'a > >, - kind: StructKind + kind: StructKind, + non_exhaustive: bool } impl< 'a > Struct< 'a > { fn new( fields: &'a syn::Fields, attrs: Vec< StructAttribute > ) -> Result< Self, syn::Error > { - collect_struct_attributes( attrs )?; + let attrs = collect_struct_attributes( attrs )?; let structure = match fields { syn::Fields::Unit => { Struct { fields: Vec::new(), - kind: StructKind::Unit + kind: StructKind::Unit, + non_exhaustive: attrs.non_exhaustive } }, syn::Fields::Named( syn::FieldsNamed { ref named, .. } ) => { Struct { fields: get_fields( named.into_iter() )?, - kind: StructKind::Named + kind: StructKind::Named, + non_exhaustive: attrs.non_exhaustive } }, syn::Fields::Unnamed( syn::FieldsUnnamed { ref unnamed, .. } ) => { Struct { fields: get_fields( unnamed.into_iter() )?, - kind: StructKind::Unnamed + kind: StructKind::Unnamed, + non_exhaustive: attrs.non_exhaustive } } }; + + if structure.non_exhaustive && structure.fields.len() > 127 { + // we forbid structs with more than 127 fields because the MSB could be used to support structs w/ more + // than 127 fields (by reading the next byte if the MSB is 1) + let message = "speedy(non_exhaustive) is not available on structs with more than 127 fields at the moment."; + return Err( syn::Error::new( fields.span(), message ) ); + } + Ok( structure ) } @@ -1314,7 +1349,17 @@ fn default_on_eof_body( body: TokenStream ) -> TokenStream { } } -fn read_field_body( field: &Field ) -> TokenStream { +fn non_exhaustive_field_body( body: TokenStream , num_field: usize) -> TokenStream { + quote! { + if _struct_size_ > #num_field { + #body? + } else { + std::default::Default::default() + } + } +} + +fn read_field_body( field: &Field, is_parent_non_exhaustive: bool, num_field: usize ) -> TokenStream { if field.skip { return quote! { std::default::Default::default() @@ -1585,10 +1630,15 @@ fn read_field_body( field: &Field ) -> TokenStream { body }; - if field.default_on_eof { - default_on_eof_body( body ) + if is_parent_non_exhaustive { + non_exhaustive_field_body( body, num_field ) } else { - quote! { #body? } + // we do not need to handle default_on_eof when parent is non_exhaustive (because default_on_eof is applied on all fields) + if field.default_on_eof { + default_on_eof_body( body ) + } else { + quote! { #body? } + } } } @@ -1596,8 +1646,16 @@ fn readable_body< 'a >( types: &mut Vec< syn::Type >, st: &Struct< 'a > ) -> (To let mut field_names = Vec::new(); let mut field_readers = Vec::new(); let mut minimum_bytes_needed = Vec::new(); + + if st.non_exhaustive { + field_readers.push( quote! { + let _struct_size_: usize = speedy::private::read_length_u64_varint( _reader_ )?; + } ); + } + + let mut num_field = 0; for field in &st.fields { - let read_value = read_field_body( field ); + let read_value = read_field_body( field, st.non_exhaustive, num_field ); let name = field.var_name(); let raw_ty = field.raw_ty; field_readers.push( quote! { let #name: #raw_ty = #read_value; } ); @@ -1607,6 +1665,16 @@ fn readable_body< 'a >( types: &mut Vec< syn::Type >, st: &Struct< 'a > ) -> (To if let Some( minimum_bytes ) = get_minimum_bytes( &field ) { minimum_bytes_needed.push( minimum_bytes ); } + + if !field.skip { + num_field += 1; + } + } + + if st.non_exhaustive && num_field <= 127 { + field_readers[0] = quote! { + let _struct_size_: usize = speedy::private::read_length_u8( _reader_ )?; + }; } let body = quote! { #(#field_readers)* }; @@ -1617,7 +1685,15 @@ fn readable_body< 'a >( types: &mut Vec< syn::Type >, st: &Struct< 'a > ) -> (To StructKind::Named => quote! { { #initializer } } }; - let minimum_bytes_needed = sum( minimum_bytes_needed ); + + + let minimum_bytes_needed = if st.non_exhaustive { + // varint64 is at least 1 byte, everything else is optional + quote! { 1 } + } else { + sum( minimum_bytes_needed ) + }; + (body, initializer, minimum_bytes_needed) } @@ -1732,11 +1808,18 @@ fn write_field_body( field: &Field ) -> TokenStream { fn writable_body< 'a >( types: &mut Vec< syn::Type >, st: &Struct< 'a > ) -> (TokenStream, TokenStream) { let mut field_names = Vec::new(); let mut field_writers = Vec::new(); + let mut field_count: usize = 0; + + if st.non_exhaustive { + field_writers.push(quote! { }); + } for field in &st.fields { if field.skip { continue; } + field_count += 1; + let write_value = write_field_body( &field ); types.extend( field.bound_types() ); @@ -1744,6 +1827,19 @@ fn writable_body< 'a >( types: &mut Vec< syn::Type >, st: &Struct< 'a > ) -> (To field_writers.push( write_value ); } + if st.non_exhaustive { + // we use u7 if less than 128 elements to optimize serialization & deserialization + let writer = if field_count > 127 { + quote! { write_length_u64_varint } + } else { + quote! { write_length_u7 } + }; + + field_writers[0] = quote! { + speedy::private::#writer(#field_count, _writer_ )?; + }; + } + let body = quote! { #(#field_writers)* }; let initializer = quote! { #(ref #field_names),* }; let initializer = match st.kind { diff --git a/tests/serialization_tests.rs b/tests/serialization_tests.rs index 1ca8a78..89c9877 100644 --- a/tests/serialization_tests.rs +++ b/tests/serialization_tests.rs @@ -2608,3 +2608,117 @@ fn test_zero_copy_cow_deserialization() { Cow::Borrowed( _ ) => panic!() } } + +#[derive(Writable, Readable, Eq, PartialEq, Debug)] +#[speedy(non_exhaustive)] +struct NonExhaustiveStructBefore { + pub field1: bool, + pub field2: Vec, + pub field3: u32, +} + +#[derive(Writable, Readable, Eq, PartialEq, Debug)] +#[speedy(non_exhaustive)] +struct NonExhaustiveStructChildBefore { + pub field1: u32, + pub field2: bool, +} + +#[derive(Writable, Readable, Eq, PartialEq, Debug)] +#[speedy(non_exhaustive)] +struct NonExhaustiveStructAfter { + pub field1: bool, + pub field2: Vec, + pub field3: u32, +} + +#[derive(Writable, Readable, Eq, PartialEq, Debug)] +#[speedy(non_exhaustive)] +struct NonExhaustiveStructChildAfter { + pub field1: u32, + pub field2: bool, + pub field_added: u8, // should always be 0 + pub field_added_2: Vec +} + +#[test] +fn test_non_exhaustive_field_appended() { + use speedy::{ + Readable, + Writable + }; + + let before = NonExhaustiveStructBefore{ + field1: true, + field2: vec![ + NonExhaustiveStructChildBefore { field1: 1, field2: false}, + NonExhaustiveStructChildBefore { field1: 2, field2: true}, + ], + field3: 9 + }; + + let buffer = before.write_to_vec().unwrap(); + assert_eq!( buffer, &[ + 3, + 1, + 2, 0, 0, 0, + 2, + 1, 0, 0, 0, + 0, + 2, + 2, 0, 0, 0, + 1, + 9, 0, 0, 0 + ]); + + let deserialized = NonExhaustiveStructAfter::read_from_buffer( &buffer ).unwrap(); + + let expectation = NonExhaustiveStructAfter{ + field1: true, + field2: vec![ + NonExhaustiveStructChildAfter { field1: 1, field2: false, field_added: 0, field_added_2: vec![]}, + NonExhaustiveStructChildAfter { field1: 2, field2: true, field_added: 0, field_added_2: vec![]}, + ], + field3: 9 + }; + assert_eq!( deserialized, expectation ); + +} + +symmetric_tests! { + non_exhaustive_struct_1 for NonExhaustiveStructBefore { + in = NonExhaustiveStructBefore{ + field1: true, + field2: vec![ + NonExhaustiveStructChildBefore { field1: 1, field2: false}, + NonExhaustiveStructChildBefore { field1: 2, field2: true}, + ], + field3: 9 + }, + le = [ + 3, + 1, + 2, 0, 0, 0, + 2, + 1, 0, 0, 0, + 0, + 2, + 2, 0, 0, 0, + 1, + 9, 0, 0, 0 + ], + be = [ + 3, + 1, + 0, 0, 0, 2, + 2, + 0, 0, 0, 1, + 0, + 2, + 0, 0, 0, 2, + 1, + 0, 0, 0, 9 + ], + minimum_bytes = 1 + } +} \ No newline at end of file