lz4 gives incorrect result after decompression

[jiale1029]

Background: Our team is recently planning to switch from go to rust and lz4 is being used to decompress the payload sent from our app. In order to ensure the switch is smooth, we need to ensure the compressed binaries can be correctly decompressed by the rust counterpart.

For this we wrapped some code that mimics our original code behaviour, which retries whenever the output buffer is insufficient.

use lz4::block;
use std::io::Result;

fn compress(src: &[u8]) -> Result<Vec<u8>> {
    return block::compress(src, None, false);
}

fn decompress(src: &[u8]) -> Result<Vec<u8>> {
    let mut factor = 3;
    loop {
        let result = block::decompress(src, Some((src.len() as i32) * factor));
        factor *= 2;
        match result {
            Ok(mut v) => {
                // truncate trailing zeros
                if let Some(i) = v.iter().rposition(|x| *x != 0) {
                    let new_len = i + 1;
                    v.truncate(new_len);
                }
                return Ok(v);
            }
            Err(e) => {
                // lz4 theoretical compression limit is 255, if buffer size exceed
                // the limit, just return result
                if factor >= 255 {
                    return Err(e);
                }
                if e.kind() != std::io::ErrorKind::InvalidData {
                    return Err(e);
                };
            }
        };
    }
}

However we noticed that, when we're testing the cases, the output compressed with rust can't get the correct result after decompression, there's always extra 1's in the rust compressed and uncompressed output.

rs:"11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", length:209
go:"111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", length:207

mod tests {
    use crate::pkg::lz4;
    use base64::prelude::*;

    #[test]
    fn compression_ratio_less_than_3() {
        let src = "2.7111111111111111111111111266249775724709369995957496696762772407663035354759457138217852516642742746".as_bytes();
        let rs_compressed = lz4::compress(&src).expect("msg");
        let go_compressed = BASE64_STANDARD.decode("TzIuNzEBAATwPDI2NjI0OTc3NTcyNDcwOTM2OTk5NTk1NzQ5NjY5Njc2Mjc3MjQwNzY2MzAzNTM1NDc1OTQ1NzEzODIxNzg1MjUxNjY0Mjc0Mjc0Ng==").expect("msg");

        let go_compressed_rs_uncompressed = lz4::decompress(go_compressed.as_slice()).expect("msg");
        assert_eq!(go_compressed_rs_uncompressed.as_slice(), src);

        let rs_compressed_rs_uncompressed = lz4::decompress(rs_compressed.as_slice()).expect("msg");
        assert_eq!(
            go_compressed_rs_uncompressed.as_slice(),
            rs_compressed_rs_uncompressed.as_slice()
        );
    }

    #[test]
    fn compression_ratio_more_than_3() {
        let src = "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111".as_bytes();
        let rs_compressed = lz4::compress(src).expect("msg");
        let go_compressed = BASE64_STANDARD
            .decode("HzEBAKkAAgDgMTExMTExMTExMTExMTE=")
            .expect("msg");

        let go_compressed_rs_uncompressed = lz4::decompress(go_compressed.as_slice()).expect("msg");
        assert_eq!(go_compressed_rs_uncompressed.as_slice(), src);

        let rs_compressed_rs_uncompressed = lz4::decompress(rs_compressed.as_slice()).expect("msg");
        println!(
            "rs:{:?}, length:{:?}",
            String::from_utf8_lossy(rs_compressed_rs_uncompressed.as_slice()),
            String::from_utf8_lossy(rs_compressed_rs_uncompressed.as_slice()).len()
        );
        println!(
            "go:{:?}, length:{:?}",
            String::from_utf8_lossy(go_compressed_rs_uncompressed.as_slice()),
            String::from_utf8_lossy(go_compressed_rs_uncompressed.as_slice()).len()
        );
        // compare the uncompressed result from rust
        assert_eq!(
            String::from_utf8_lossy(rs_compressed_rs_uncompressed.as_slice()),
            String::from_utf8_lossy(src),
        );
    }
}