Implement matching compression for old gzip versions (#22)

* Implement matching compression for old gzip versions

* Add gzip tests

* Add CHANGELOG entry

* Avoid typing.Optional

* Add references to README

CHANGELOG.md

@@ -7,16 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Add matching zlib/DEFLATE compression for old gzip versions.
+
 ## [0.4.0] - 2024-06-03
 
 ### Added
 
-- Add MIO0 compression to CLI (#17)
+- Add MIO0 compression to CLI.
 
 ### Changed
 
-- Speed up compression by 2100% (#18)
-- Move CompressionType from library to CLI (#19)
+- Speed up compression by 2100%.
+- Move CompressionType from library to CLI.
 
 ## [0.3.1] - 2024-01-20
 

README.md

@@ -64,3 +64,6 @@ to check the latest release.
   - Reference implementation by Mr-Wiseguy: <https://gist.github.com/Mr-Wiseguy/6cca110d74b32b5bb19b76cfa2d7ab4f>
 - MIO0
   - Hack64.net docs: <https://hack64.net/wiki/doku.php?id=super_mario_64:mio0>
+- gzip
+  - Reference implementation: <https://github.com/Thar0/gzip-1.3.3-ique>
+  - DEFLATE specification: <https://datatracker.ietf.org/doc/html/rfc1951>

c_bindings/include/crunch64.h

@@ -6,5 +6,6 @@
 #include "crunch64/mio0.h"
 #include "crunch64/yay0.h"
 #include "crunch64/yaz0.h"
+#include "crunch64/gzip.h"
 
 #endif

c_bindings/include/crunch64/error.h

@@ -18,6 +18,7 @@ typedef enum Crunch64Error {
     Crunch64Error_ByteConversion,
     Crunch64Error_OutOfBounds,
     Crunch64Error_NullPointer,
+    Crunch64Error_InvalidCompressionLevel,
 } Crunch64Error;
 
 #ifdef __cplusplus

c_bindings/include/crunch64/gzip.h

@@ -0,0 +1,58 @@
+#ifndef CRUNCH64_GZIP_H
+#define CRUNCH64_GZIP_H
+#pragma once
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "error.h"
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+/**
+ * @brief Get a size big enough to allocate a buffer that can fit the compressed data produced by compressing `src`.
+ *
+ * Returning `true` means the function succeeded and the requested size was put in `dst_size`
+ *
+ * If this function fails to calculate said size then it will return `false` and `dst_size` may not be a valid value.
+ *
+ * @param dst_size[out] Will be set to the requested size.
+ * @param src_len Size of `src`
+ * @param src[in] Data that would be compressed
+ */
+Crunch64Error crunch64_gzip_compress_bound(size_t *dst_size, size_t src_len, const uint8_t *const src);
+
+/**
+ * @brief Compresses the data pointed by `src` and puts that data into `dst`.
+ *
+ * The `dst` should point to a buffer big enough to hold the compressed data. To know how big said buffer must be
+ * refer to `crunch64_gzip_compress_bound`.
+ *
+ * When this function is called, `dst_len` must point to the size of the `dst` pointer, allowing for range checking
+ * and avoiding to write out of bounds.
+ *
+ * If the function succeedes it returns `true` and it puts the compressed data on `dst` and the actual compressed
+ * size is put on `dst_len`.
+ *
+ * If this function fails it will return `false`. `dst_size` and `dst` may have garbage data.
+ *
+ * `dst` will include the gzip footer but no gzip header.
+ *
+ * @param dst_len[in,out] Will be set to the compressed size. It should point to the size of the `dst` buffer when the function is called.
+ * @param dst[out] Pointer to buffer big enough to hold the compressed data.
+ * @param src_len The length of the data pointed by `src`.
+ * @param src[in] Pointer to the decompressed data.
+ * @param level Compression level (4-9).
+ * @param small_mem If `true` then the function will output compressed blocks more often.
+ */
+Crunch64Error crunch64_gzip_compress(size_t *dst_len, uint8_t *dst, size_t src_len, const uint8_t *const src, int level, bool small_mem);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

c_bindings/tests.c

@@ -220,10 +220,10 @@ void run_tests(const char *name, const char *file_extension, compress_bound_fn c
         assert(compressed_data_size > 0);
         assert(compressed_data != NULL);
 
-        if (!test_matching_decompression(decompress_bound, decompress, bin_size, bin, compressed_data_size, compressed_data)) {
+        if (decompress != NULL && !test_matching_decompression(decompress_bound, decompress, bin_size, bin, compressed_data_size, compressed_data)) {
             errors++;
         }
-        if (!test_matching_compression(compress_bound, compress, bin_size, bin, compressed_data_size, compressed_data)) {
+        if (compress != NULL && !test_matching_compression(compress_bound, compress, bin_size, bin, compressed_data_size, compressed_data)) {
             errors++;
         }
 
@@ -240,11 +240,30 @@ void run_tests(const char *name, const char *file_extension, compress_bound_fn c
     }
 }
 
+int gzip_level;
+bool gzip_small_mem;
+
+Crunch64Error gzip_compress(size_t *dst_size, uint8_t *dst, size_t src_size, const uint8_t *src) {
+    return crunch64_gzip_compress(dst_size, dst, src_size, src, gzip_level, gzip_small_mem);
+}
+
 int main(void) {
     run_tests("yay0", ".Yay0", crunch64_yay0_compress_bound, crunch64_yay0_compress, crunch64_yay0_decompress_bound, crunch64_yay0_decompress);
     run_tests("yaz0", ".Yaz0", crunch64_yaz0_compress_bound, crunch64_yaz0_compress, crunch64_yaz0_decompress_bound, crunch64_yaz0_decompress);
     run_tests("mio0", ".MIO0", crunch64_mio0_compress_bound, crunch64_mio0_compress, crunch64_mio0_decompress_bound, crunch64_mio0_decompress);
 
+    gzip_level = 9;
+    gzip_small_mem = false;
+    run_tests("gzip (level 9)", ".gzip-9", crunch64_gzip_compress_bound, gzip_compress, NULL, NULL);
+
+    gzip_level = 9;
+    gzip_small_mem = true;
+    run_tests("gzip (level 9, small_mem)", ".gzip-9-small-mem", crunch64_gzip_compress_bound, gzip_compress, NULL, NULL);
+
+    gzip_level = 6;
+    gzip_small_mem = true;
+    run_tests("gzip (level 6, small_mem)", ".gzip-6-small-mem", crunch64_gzip_compress_bound, gzip_compress, NULL, NULL);
+
     if (errors == 0) {
         fprintf(stderr, "All tests passed\n");
         return 0;

cli/src/main.rs

@@ -18,6 +18,7 @@ enum CompressionType {
     Yay0,
     Yaz0,
     Mio0,
+    Gzip,
 }
 
 #[derive(Parser, Debug)]
@@ -31,40 +32,47 @@ struct Args {
     in_path: String,
     #[arg()]
     out_path: String,
+    /// Compression level for gzip (4-9)
+    #[arg(long, default_value_t = 9)]
+    level: usize,
+    /// Output gzip blocks more frequently
+    #[arg(long)]
+    small_mem: bool,
 }
 
-fn compress(format: CompressionType, bytes: &[u8]) -> Result<Box<[u8]>, Crunch64Error> {
-    match format {
+fn compress(args: &Args, bytes: &[u8]) -> Result<Box<[u8]>, Crunch64Error> {
+    match args.format {
         CompressionType::Yay0 => crunch64::yay0::compress(bytes),
         CompressionType::Yaz0 => crunch64::yaz0::compress(bytes),
         CompressionType::Mio0 => crunch64::mio0::compress(bytes),
+        CompressionType::Gzip => crunch64::gzip::compress(bytes, args.level, args.small_mem),
         // _ => Err(Crunch64Error::UnsupportedCompressionType),
     }
 }
 
-fn decompress(format: CompressionType, bytes: &[u8]) -> Result<Box<[u8]>, Crunch64Error> {
-    match format {
+fn decompress(args: &Args, bytes: &[u8]) -> Result<Box<[u8]>, Crunch64Error> {
+    match args.format {
         CompressionType::Yay0 => crunch64::yay0::decompress(bytes),
         CompressionType::Yaz0 => crunch64::yaz0::decompress(bytes),
         CompressionType::Mio0 => crunch64::mio0::decompress(bytes),
-        //_ => Err(Crunch64Error::UnsupportedCompressionType),
+        _ => Err(Crunch64Error::UnsupportedCompressionType),
     }
 }
 
 fn main() {
     let args = Args::parse();
 
-    let file_bytes = read_file_bytes(args.in_path);
+    let file_bytes = read_file_bytes(&args.in_path);
 
     let out_bytes = match args.command {
-        Command::Compress => match compress(args.format, file_bytes.as_slice()) {
+        Command::Compress => match compress(&args, file_bytes.as_slice()) {
             Ok(bytes) => bytes,
             Err(error) => {
                 eprintln!("{:?}", error);
                 process::exit(1);
             }
         },
-        Command::Decompress => match decompress(args.format, file_bytes.as_slice()) {
+        Command::Decompress => match decompress(&args, file_bytes.as_slice()) {
             Ok(bytes) => bytes,
             Err(error) => {
                 eprintln!("{:?}", error);

lib/Cargo.toml

@@ -13,6 +13,7 @@ path = "src/lib.rs"
 crate-type = ["lib", "staticlib", "cdylib"]
 
 [dependencies]
+crc32fast = "1.4.2"
 pyo3 = { version="0.20.0", features = ["extension-module"], optional = true }
 thiserror = "1.0"
 

lib/crunch64/__init__.py

@@ -10,3 +10,4 @@
 from . import yay0 as yay0
 from . import yaz0 as yaz0
 from . import mio0 as mio0
+from . import gzip as gzip

lib/crunch64/crunch64.pyi

@@ -5,3 +5,4 @@ from __future__ import annotations
 from . import yay0 as yay0
 from . import yaz0 as yaz0
 from . import mio0 as mio0
+from . import gzip as gzip

lib/crunch64/gzip.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+from .crunch64 import compress_gzip as compress

lib/crunch64/gzip.pyi

@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+def compress(data: bytes, level: int = 9, small_mem: bool = False) -> bytes: ...