Add MIO0 compression and decompression (#10)

* Implement mio0 decompression

* Add mio0 test cases

* fmt and clippy

* a

* Implement mio0 compression

* Forgot to add tests to CI

* rewrite mio0 to make it more similar to yay0

* version bump

* cli version

.github/workflows/c_bindings.yml

@@ -19,6 +19,9 @@ jobs:
       - name: Make C test programs
         run: make -C c_bindings BUILD_MODE=release
 
+      - name: Test MIO0
+        run: ./c_bindings/tests/test_mio0.sh
+
       - name: Test Yay0
         run: ./c_bindings/tests/test_yay0.sh
 

.github/workflows/python_bindings.yml

@@ -21,6 +21,9 @@ jobs:
     - name: Install local crunch64
       run: python3 -m pip install ./lib
 
+    - name: Test MIO0
+      run: python3 ./python_bindings/tests/test_mio0.py
+
     - name: Test Yay0
       run: python3 ./python_bindings/tests/test_yay0.py
 

README.md

@@ -48,3 +48,10 @@ python3 -m pip install "git+https://github.com/decompals/crunch64.git#egg=crunch
 ```
 
 NOTE: Installing the development version is not recommended unless you know what you are doing. Proceed at your own risk.
+
+## References
+
+- Yaz0
+  - Reference implementation by Mr-Wiseguy: <https://gist.github.com/Mr-Wiseguy/6cca110d74b32b5bb19b76cfa2d7ab4f>
+- MIO0
+  - Hack64.net docs: <https://hack64.net/wiki/doku.php?id=super_mario_64:mio0>

c_bindings/include/crunch64.h

@@ -3,6 +3,7 @@
 #pragma once
 
 #include "crunch64/error.h"
+#include "crunch64/mio0.h"
 #include "crunch64/yay0.h"
 #include "crunch64/yaz0.h"
 

c_bindings/include/crunch64/mio0.h

@@ -0,0 +1,94 @@
+#ifndef CRUNCH64_MIO0_H
+#define CRUNCH64_MIO0_H
+#pragma once
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "error.h"
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+/**
+ * @brief Get a size big enough to allocate a buffer that can fit the uncompressed data produced by uncompressing `src`.
+ *
+ * The compressed data must include the Mio0 header.
+ *
+ * Returning `true` means the function succeeded and the requested size was put in `dst_size`.
+ *
+ * If this function fails to calculate said size then it will return `false` and `dst_size` may have garbage data.
+ *
+ * @param dst_size[out] Will be set to the requested size.
+ * @param src_len Size of `src`
+ * @param src[in] Compressed Mio0 data
+ */
+Crunch64Error crunch64_mio0_decompress_bound(size_t *dst_size, size_t src_len, const uint8_t *const src);
+
+/**
+ * @brief Decompresses the data pointed by `src` and puts that data into `dst`.
+ *
+ * The `dst` should point to a buffer big enough to hold the decompressed data. To know how big said buffer must be
+ * refer to `crunch64_mio0_decompress_bound`.
+ *
+ * When this function is called, `dst_len` must point to the size of the `dst` pointer, allowing for range checking
+ * and avoiding to write out of bounds.
+ *
+ * If the function succeedes it returns `true` and it puts the decompressed data on `dst` and the actual decompressed
+ * size is put on `dst_len`.
+ *
+ * If this function fails it will return `false`. `dst_size` and `dst` may have garbage data.
+ *
+ * @param dst_len[in,out] Will be set to the decompressed size. It should point to the size of the `dst` buffer when the function is called.
+ * @param dst[out] Pointer to buffer big enough to hold the decompressed data.
+ * @param src_len The length of the data pointed by `src`.
+ * @param src[in] Pointer to compressed data. Must contain the Mio0 header.
+ */
+Crunch64Error crunch64_mio0_decompress(size_t *dst_len, uint8_t *dst, size_t src_len, const uint8_t *const src);
+
+/**
+ * @brief Get a size big enough to allocate a buffer that can fit the compressed data produced by compressing `src`.
+ *
+ * The compressed data must include the Mio0 header.
+ *
+ * Returning `true` means the function succeeded and the requested size was put in `dst_size`
+ *
+ * If this function fails to calculate said size then it will return `false` and `dst_size` may not be a valid value.
+ *
+ * @param dst_size[out] Will be set to the requested size.
+ * @param src_len Size of `src`
+ * @param src[in] Data that would be compressed
+ */
+Crunch64Error crunch64_mio0_compress_bound(size_t *dst_size, size_t src_len, const uint8_t *const src);
+
+/**
+ * @brief Compresses the data pointed by `src` and puts that data into `dst`.
+ *
+ * The `dst` should point to a buffer big enough to hold the compressed data. To know how big said buffer must be
+ * refer to `crunch64_mio0_compress_bound`.
+ *
+ * When this function is called, `dst_len` must point to the size of the `dst` pointer, allowing for range checking
+ * and avoiding to write out of bounds.
+ *
+ * If the function succeedes it returns `true` and it puts the compressed data on `dst` and the actual compressed
+ * size is put on `dst_len`.
+ *
+ * If this function fails it will return `false`. `dst_size` and `dst` may have garbage data.
+ *
+ * `dst` will include the Mio0 header.
+ *
+ * @param dst_len[in,out] Will be set to the compressed size. It should point to the size of the `dst` buffer when the function is called.
+ * @param dst[out] Pointer to buffer big enough to hold the compressed data.
+ * @param src_len The length of the data pointed by `src`.
+ * @param src[in] Pointer to the decompressed data.
+ */
+Crunch64Error crunch64_mio0_compress(size_t *dst_len, uint8_t *dst, size_t src_len, const uint8_t *const src);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

c_bindings/tests/test_mio0.c

@@ -0,0 +1,137 @@
+#include "crunch64.h"
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "utils.h"
+
+bool decompress(size_t *dst_size, uint8_t **dst, size_t src_size, const uint8_t *src)
+{
+    size_t decompressed_size;
+    uint8_t *decompressed_data = NULL;
+
+    Crunch64Error size_request_ok = crunch64_mio0_decompress_bound(&decompressed_size, src_size, src);
+    if (size_request_ok != Crunch64Error_Okay)
+    {
+        fprintf(stderr, " failed to request size for buffer. Reason: %s\n", get_crunch64_error_str(size_request_ok));
+        return false;
+    }
+
+    decompressed_data = malloc(decompressed_size * sizeof(uint8_t));
+    if (decompressed_data == NULL)
+    {
+        fprintf(stderr, " malloc fail: 0x%zX bytes\n", decompressed_size * sizeof(uint8_t));
+        return false;
+    }
+
+    Crunch64Error decompress_ok = crunch64_mio0_decompress(&decompressed_size, decompressed_data, src_size, src);
+    if (decompress_ok != Crunch64Error_Okay)
+    {
+        fprintf(stderr, " failed to decompress data. Reason: %s\n", get_crunch64_error_str(decompress_ok));
+        free(decompressed_data);
+        return false;
+    }
+
+    *dst_size = decompressed_size;
+    *dst = decompressed_data;
+
+    fprintf(stderr, " OK\n");
+    return true;
+}
+
+bool compress(size_t *dst_size, uint8_t **dst, size_t src_size, const uint8_t *src)
+{
+    size_t compressed_size;
+    uint8_t *compressed_data = NULL;
+
+    assert(dst_size != NULL);
+    assert(dst != NULL);
+    assert(src != NULL);
+
+    Crunch64Error size_request_ok = crunch64_mio0_compress_bound(&compressed_size, src_size, src);
+    if (size_request_ok != Crunch64Error_Okay)
+    {
+        fprintf(stderr, " failed to request size for buffer. Reason: %s\n", get_crunch64_error_str(size_request_ok));
+        return false;
+    }
+
+    compressed_data = malloc(compressed_size * sizeof(uint8_t));
+    if (compressed_data == NULL)
+    {
+        fprintf(stderr, " malloc fail: 0x%zX bytes\n", compressed_size * sizeof(uint8_t));
+        return false;
+    }
+
+    Crunch64Error compress_ok = crunch64_mio0_compress(&compressed_size, compressed_data, src_size, src);
+    if (compress_ok != Crunch64Error_Okay)
+    {
+        fprintf(stderr, " failed to decompress data. Reason: %s\n", get_crunch64_error_str(compress_ok));
+        free(compressed_data);
+        return false;
+    }
+
+    *dst_size = compressed_size;
+    *dst = compressed_data;
+
+    fprintf(stderr, " OK\n");
+
+    return true;
+}
+
+void print_usage(int argc, char *argv[])
+{
+    (void)argc;
+
+    fprintf(stderr, "Usage: %s bin_file compressed_file\n", argv[0]);
+    fprintf(stderr, "\n");
+    fprintf(stderr, "This programs tests compression and decompression produces matching output\n");
+}
+
+int main(int argc, char *argv[])
+{
+    int ret = 0;
+
+    if (argc < 2)
+    {
+        print_usage(argc, argv);
+        return -1;
+    }
+
+    const char *bin_path = argv[1];
+    const char *compressed_path = argv[2];
+
+    fprintf(stderr, "Reading file %s\n", bin_path);
+    size_t bin_size = 0;
+    uint8_t *bin = read_binary_file(bin_path, &bin_size);
+    assert(bin_size > 0);
+    assert(bin != NULL);
+
+    fprintf(stderr, "Reading file %s\n", compressed_path);
+    size_t compressed_data_size = 0;
+    uint8_t *compressed_data = read_binary_file(compressed_path, &compressed_data_size);
+    assert(compressed_data_size > 0);
+    assert(compressed_data != NULL);
+
+    if (!test_matching_decompression(bin_size, bin, compressed_data_size, compressed_data))
+    {
+        ret++;
+    }
+    if (!test_matching_compression(bin_size, bin, compressed_data_size, compressed_data))
+    {
+        ret++;
+    }
+    if (!test_cycle_decompressed(bin_size, bin))
+    {
+        ret++;
+    }
+    if (!test_cycle_compressed(compressed_data_size, compressed_data))
+    {
+        ret++;
+    }
+
+    free(bin);
+    free(compressed_data);
+
+    return ret;
+}

c_bindings/tests/test_mio0.sh

@@ -0,0 +1,14 @@
+#! /usr/bin/env bash
+
+set -e
+
+for i in test_data/*.MIO0; do
+    [ -f "$i" ] || break
+    echo "Processing:" $i
+
+    # Remove the extension
+    BIN_PATH=$(echo $i | sed 's/.MIO0//')
+
+    c_bindings/tests/test_mio0.elf $BIN_PATH $i
+    echo
+done

cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "crunch64-cli"
-version = "0.1.1"
+version = "0.2.0"
 edition = "2021"
 description = "A utility for compressing/decompressing files with common n64 formats"
 repository = "https://github.com/decompals/crunch64"
@@ -11,5 +11,5 @@ name = "crunch64"
 path = "src/main.rs"
 
 [dependencies]
-crunch64 = { version = "0.1.1", path = "../lib" }
+crunch64 = { version = "0.2.0", path = "../lib" }
 clap = { version = "4.4.11", features = ["derive"] }

cli/src/main.rs

@@ -33,7 +33,7 @@ fn main() {
     let compression_format = match args.format.as_str() {
         "Yay0" | "yay0" => CompressionType::Yay0,
         "Yaz0" | "yaz0" => CompressionType::Yaz0,
-        "Mio0" | "mio0" => CompressionType::Mio0,
+        "MIO0" | "Mio0" | "mio0" => CompressionType::Mio0,
         _ => {
             let mut cmd = Args::command();
             cmd.error(

lib/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "crunch64"
 # Version should be synced with lib/pyproject.toml and lib/crunch64/__init__.py
-version = "0.1.1"
+version = "0.2.0"
 edition = "2021"
 description = "A library for handling common compression formats for N64 games"
 repository = "https://github.com/decompals/crunch64"

lib/crunch64/__init__.py

@@ -3,9 +3,10 @@
 from __future__ import annotations
 
 # Version should be synced with lib/Cargo.toml and lib/pyproject.toml
-__version_info__ = (0, 1, 1)
+__version_info__ = (0, 2, 0)
 __version__ = ".".join(map(str, __version_info__))
 __author__ = "decompals"
 
 from . import yay0 as yay0
 from . import yaz0 as yaz0
+from . import mio0 as mio0

lib/crunch64/crunch64.pyi

@@ -4,3 +4,4 @@ from __future__ import annotations
 
 from . import yay0 as yay0
 from . import yaz0 as yaz0
+from . import mio0 as mio0

lib/crunch64/mio0.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+from .crunch64 import decompress_mio0 as decompress
+from .crunch64 import compress_mio0 as compress

lib/crunch64/mio0.pyi

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+def decompress(data: bytes) -> bytes: ...
+def compress(data: bytes) -> bytes: ...

lib/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "crunch64"
 # Version should be synced with lib/Cargo.toml and lib/crunch64/__init__.py
-version = "0.1.1"
+version = "0.2.0"
 description = "A library for handling common compression formats for N64 games"
 requires-python = ">=3.7"
 dependencies = [