use base 127 encoding for size_hint in the header to allow for future…

… header fields and also to simplify a compliant header parser so it does not need to understand the bit field formats of the window_size of brotli

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "brotli"
-version = "3.1.6"
+version = "3.1.7"
 authors = ["Daniel Reiter Horn <[email protected]>", "The Brotli Authors"]
 description = "A brotli compressor and decompressor that with an interface avoiding the rust stdlib. This makes it suitable for embedded devices and kernels. It is designed with a pluggable allocator so that the standard lib's allocator may be employed. The default build also includes a stdlib allocator and stream interface. Disable this with --features=no-stdlib. All included code is safe."
 license = "BSD-3-Clause/MIT"

src/bin/test_threading.rs

@@ -83,11 +83,11 @@ fn multi_threaded_split_compression_test_4() {
 }
 #[test]
 fn multi_threaded_split_compression_test_5() {
-    multi_threaded_split_compression_test(&RANDOM_THEN_UNICODE[..], 5, 9, false, 139125)
+    multi_threaded_split_compression_test(&RANDOM_THEN_UNICODE[..], 5, 9, false, 139126)
 }
 #[test]
 fn multi_threaded_split_compression_test_1b1() {
-    multi_threaded_split_compression_test(&RANDOM_THEN_UNICODE[..1], 5, 9, false, 139125)
+    multi_threaded_split_compression_test(&RANDOM_THEN_UNICODE[..1], 5, 9, false, 139126)
 }
 #[test]
 fn multi_threaded_split_compression_test_1b5() {

src/enc/brotli_bit_stream.rs

@@ -2776,35 +2776,64 @@ pub fn BrotliWriteEmptyLastMetaBlock(storage_ix: &mut usize, storage: &mut [u8])
     BrotliWriteBits(1u8, 1u64, storage_ix, storage);
     JumpToByteBoundary(storage_ix, storage);
 }
+
+const MAX_SIZE_ENCODING:usize = 10;
+
+fn encode_base_128(mut value: u64)-> (usize, [u8;MAX_SIZE_ENCODING]) {
+  let mut ret = [0u8; MAX_SIZE_ENCODING];
+  for index in 0..ret.len() {
+    ret[index] = (value & 0x7f) as u8;
+    value >>= 7;
+    if value != 0 {
+      ret[index] |= 0x80;
+    } else {
+      return (index + 1, ret)
+    }
+  }
+  (ret.len(), ret)
+}
+
 pub fn BrotliWriteMetadataMetaBlock(params: &BrotliEncoderParams, storage_ix: &mut usize, storage: &mut [u8]) {
     BrotliWriteBits(1u8, 0u64, storage_ix, storage); // not last
     BrotliWriteBits(2u8, 3u64, storage_ix, storage); // MNIBBLES = 0 (pattern 1,1)
     BrotliWriteBits(1u8, 0u64, storage_ix, storage); // reserved
     BrotliWriteBits(2u8, 1u64, storage_ix, storage); // num bytes for length of magic number header
-    let log_size_hint_bits = 64 - (params.size_hint as u64).leading_zeros();
-    let size_hint_bytes = (log_size_hint_bits as u64 + 7) >> 3;
+    let (size_hint_count, size_hint_b128) = encode_base_128(params.size_hint as u64);
 
-    BrotliWriteBits(8u8, 3 + size_hint_bytes, storage_ix, storage); // 1 byte of data: writing 12 for the magic number header
+    BrotliWriteBits(8u8, 3 + size_hint_count as u64, storage_ix, storage); // 1 byte of data: writing 12 for the magic number header
     JumpToByteBoundary(storage_ix, storage);
-    let magic_number = if params.catable && !params.use_dictionary {
+    let magic_number: [u8;3] = if params.catable && !params.use_dictionary {
         [0xe1, 0x97, 0x81]
     } else if params.appendable {
         [0xe1, 0x97, 0x82]
     } else {
         [0xe1, 0x97, 0x80]
     };
-    let size_hint_64 = params.size_hint as u64;
-    let header = [magic_number[0], magic_number[1], magic_number[2], VERSION,
-                        (size_hint_64 & 0xff) as u8,
-                        ((size_hint_64 >> 8) & 0xff) as u8,
-                        ((size_hint_64 >> 16) & 0xff) as u8,
-                        ((size_hint_64 >> 24) & 0xff) as u8,
-                        ((size_hint_64 >> 32) & 0xff) as u8,
-                        ((size_hint_64 >> 40) & 0xff) as u8,
-                        ((size_hint_64 >> 48) & 0xff) as u8,
-                        ((size_hint_64 >> 56) & 0xff) as u8,
-    ];
-    for magic in header[..4 + size_hint_bytes as usize].iter() {
-        BrotliWriteBits(8u8, u64::from(*magic), storage_ix, storage);
+    for magic in magic_number.iter() {
+      BrotliWriteBits(8u8, u64::from(*magic), storage_ix, storage);
+    }
+    BrotliWriteBits(8u8, u64::from(VERSION), storage_ix, storage);
+    for sh in size_hint_b128[..size_hint_count].iter() {
+        BrotliWriteBits(8u8, u64::from(*sh), storage_ix, storage);
     }
 }
+
+mod test {
+  use super::{encode_base_128, MAX_SIZE_ENCODING};
+  #[test]
+  fn test_encode_base_128() {
+    assert_eq!(encode_base_128(0), (1,[0u8;MAX_SIZE_ENCODING]));
+    assert_eq!(encode_base_128(1), (1,[1,0,0,0,0,0,0,0,0,0]));
+    assert_eq!(encode_base_128(127), (1,[0x7f,0,0,0,0,0,0,0,0,0]));
+    assert_eq!(encode_base_128(128), (2,[0x80,0x1,0,0,0,0,0,0,0,0]));
+    assert_eq!(encode_base_128(16383), (2,[0xff,0x7f,0,0,0,0,0,0,0,0]));
+    assert_eq!(encode_base_128(16384), (3,[0x80,0x80,0x1,0,0,0,0,0,0,0]));
+    assert_eq!(encode_base_128(2097151), (3,[0xff,0xff,0x7f,0,0,0,0,0,0,0]));
+    assert_eq!(encode_base_128(2097152), (4,[0x80,0x80,0x80,0x1,0,0,0,0,0,0]));
+    assert_eq!(encode_base_128(4194303), (4,[0xff,0xff,0xff,0x1,0,0,0,0,0,0]));
+    assert_eq!(encode_base_128(4294967295), (5,[0xff,0xff,0xff,0xff,0xf,0,0,0,0,0]));
+    assert_eq!(encode_base_128(4294967296), (5,[0x80,0x80,0x80,0x80,0x10,0,0,0,0,0]));
+    assert_eq!(encode_base_128(9223372036854775808), (10,[0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x1]));
+    assert_eq!(encode_base_128(18446744073709551615), (10,[0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x1]));
+  }
+}

src/enc/encode.rs

@@ -3009,7 +3009,7 @@ pub fn BrotliEncoderTakeOutput<'a, Alloc:BrotliAlloc>(s: &'a mut BrotliEncoderSt
 
 
 pub fn BrotliEncoderVersion() -> u32 {
-  0x1000f00u32
+  0x1000f01u32
 }