CRC big endian support

.github/workflows/ci.yml

@@ -232,6 +232,7 @@ jobs:
         id: test
         uses: cross-platform-actions/[email protected]
         with:
+          #note: sanitizer seems to be broken on s390x (fails trying to allocate several tbs of mem) and package manager works slightly differently that on regular ubuntu
           operating_system: freebsd
           architecture: x86-64
           version: '14.0'
@@ -243,6 +244,30 @@ jobs:
             chmod a+x builder
             ./builder build -p ${{ env.PACKAGE_NAME }}
 
+  s390x: #big-endian
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: uraimo/run-on-arch-action@v2
+        name: Run commands
+        id: runcmd
+        with:
+          arch: s390x
+          distro: ubuntu22.04
+          install: |
+            apt-get update -q -y
+            apt-get -y install sudo
+            apt-get -y install cmake
+            apt-get -y install make
+            apt-get -y install g++
+            apt-get -y install python3
+            apt-get -y install git
+          run: |
+            lscpu | grep Endian
+            python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
+            chmod a+x builder
+            ./builder build -p ${{ env.PACKAGE_NAME }} --variant s390x --cmake-extra=-DENABLE_SANITIZERS=OFF
+
   openbsd:
     runs-on: ubuntu-24.04 # latest
     strategy:

builder.json

@@ -10,5 +10,15 @@
     "test_steps": [
         "test",
         "{install_dir}/bin/checksum-profile{exe}"
-    ]
+    ],
+
+    "variants": {
+        "s390x": {
+            "hosts": {
+                "ubuntu": {
+                    "!pkg_setup": []
+                }
+            }
+        }
+    }
 }

include/aws/checksums/private/crc_util.h

@@ -5,6 +5,7 @@
  * SPDX-License-Identifier: Apache-2.0.
  */
 
+#include <aws/common/byte_order.h>
 #include <aws/common/stdint.h>
 #include <limits.h>
 
@@ -21,4 +22,32 @@
         return val;                                                                                                    \
     }
 
+/* helper function to reverse byte order on big-endian platforms*/
+static inline uint32_t aws_swap_bytes_if_needed_32(uint32_t x) {
+    if (!aws_is_big_endian()) {
+        return x;
+    }
+
+    uint8_t c1 = x & 0xFF;
+    uint8_t c2 = (x >> 8) & 0xFF;
+    uint8_t c3 = (x >> 16) & 0xFF;
+    uint8_t c4 = (x >> 24) & 0xFF;
+
+    return ((uint32_t)c1 << 24) + ((uint32_t)c2 << 16) + ((uint32_t)c3 << 8) + c4;
+}
+
+/* Reverse the bytes in a 64-bit word. */
+static inline uint64_t aws_swap_bytes_if_needed_64(uint64_t x) {
+    if (!aws_is_big_endian()) {
+        return x;
+    }
+
+    uint64_t m;
+    m = 0xff00ff00ff00ff;
+    x = ((x >> 8) & m) | (x & m) << 8;
+    m = 0xffff0000ffff;
+    x = ((x >> 16) & m) | (x & m) << 16;
+    return x >> 32 | x << 32;
+}
+
 #endif /* AWS_CHECKSUMS_PRIVATE_CRC_UTIL_H */

source/crc_sw.c

@@ -3,6 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0.
  */
 #include <aws/checksums/private/crc_priv.h>
+#include <aws/checksums/private/crc_util.h>
 #include <stddef.h>
 
 /* The Ethernet, gzip, et.al CRC32 polynomial (reverse of 0x04C11DB7) */
@@ -1149,7 +1150,7 @@ static uint32_t s_crc_generic_sb4(const uint8_t *input, int length, uint32_t crc
     uint32_t(*table)[16][256] = (uint32_t(*)[16][256])table_ptr;
 
     while (remaining >= 4) {
-        crc ^= *current++;
+        crc ^= aws_swap_bytes_if_needed_32(*current++);
         crc = (*table)[3][crc & 0xff] ^ (*table)[2][(crc >> 8) & 0xff] ^ (*table)[1][(crc >> 16) & 0xff] ^
               (*table)[0][crc >> 24];
         remaining -= 4;
@@ -1165,8 +1166,8 @@ static uint32_t s_crc_generic_sb8(const uint8_t *input, int length, uint32_t crc
     uint32_t(*table)[16][256] = (uint32_t(*)[16][256])table_ptr;
 
     while (remaining >= 8) {
-        uint32_t c1 = *current++ ^ crc;
-        uint32_t c2 = *current++;
+        uint32_t c1 = aws_swap_bytes_if_needed_32(*current++) ^ crc;
+        uint32_t c2 = aws_swap_bytes_if_needed_32(*current++);
         uint32_t t1 = (*table)[7][c1 & 0xff] ^ (*table)[6][(c1 >> 8) & 0xff] ^ (*table)[5][(c1 >> 16) & 0xff] ^
                       (*table)[4][(c1 >> 24) & 0xff];
         uint32_t t2 = (*table)[3][c2 & 0xff] ^ (*table)[2][(c2 >> 8) & 0xff] ^ (*table)[1][(c2 >> 16) & 0xff] ^
@@ -1185,10 +1186,10 @@ static uint32_t s_crc_generic_sb16(const uint8_t *input, int length, uint32_t cr
     uint32_t(*table)[16][256] = (uint32_t(*)[16][256])table_ptr;
 
     while (remaining >= 16) {
-        uint32_t c1 = *current++ ^ crc;
-        uint32_t c2 = *current++;
-        uint32_t c3 = *current++;
-        uint32_t c4 = *current++;
+        uint32_t c1 = aws_swap_bytes_if_needed_32(*current++) ^ crc;
+        uint32_t c2 = aws_swap_bytes_if_needed_32(*current++);
+        uint32_t c3 = aws_swap_bytes_if_needed_32(*current++);
+        uint32_t c4 = aws_swap_bytes_if_needed_32(*current++);
         uint32_t t1 = (*table)[15][c1 & 0xff] ^ (*table)[14][(c1 >> 8) & 0xff] ^ (*table)[13][(c1 >> 16) & 0xff] ^
                       (*table)[12][(c1 >> 24) & 0xff];
         uint32_t t2 = (*table)[11][c2 & 0xff] ^ (*table)[10][(c2 >> 8) & 0xff] ^ (*table)[9][(c2 >> 16) & 0xff] ^

tests/crc64_test.c

@@ -5,6 +5,8 @@
 
 #include <aws/checksums/crc.h>
 #include <aws/checksums/private/crc64_priv.h>
+#include <aws/checksums/private/crc_util.h>
+#include <aws/common/encoding.h>
 #include <aws/testing/aws_test_harness.h>
 
 // The polynomial used for CRC64NVME (in bit-reflected form)
@@ -54,7 +56,8 @@ static int s_test_known_crc(
     ASSERT_HEX_EQUALS(expected_crc, result, "%s(%s)", func_name, data_name);
 
     // Compute the residue of the buffer (the CRC of the buffer plus its CRC) - will always be a constant value
-    uint64_t residue = func((const uint8_t *)&result, 8, result); // assuming little endian
+    uint64_t result_swapped = aws_swap_bytes_if_needed_64(result);
+    uint64_t residue = func((const uint8_t *)&result_swapped, 8, result); // assuming little endian
     ASSERT_HEX_EQUALS(expected_residue, residue, "len %d residue %s(%s)", length, func_name, data_name);
 
     // chain the crc computation so 2 calls each operate on about 1/2 of the buffer

tests/crc_test.c

@@ -5,6 +5,7 @@
 
 #include <aws/checksums/crc.h>
 #include <aws/checksums/private/crc_priv.h>
+#include <aws/checksums/private/crc_util.h>
 
 #include <aws/common/device_random.h>
 
@@ -39,7 +40,6 @@ typedef uint32_t(crc_fn)(const uint8_t *input, int length, uint32_t previousCrc3
 
 // Slow reference implementation that computes a 32-bit bit-reflected/bit-inverted CRC using the provided polynomial.
 static uint32_t s_crc_32_reference(const uint8_t *input, int length, const uint32_t previousCrc, uint32_t polynomial) {
-
     uint32_t crc = ~previousCrc;
     while (length-- > 0) {
         crc ^= *input++;
@@ -73,8 +73,9 @@ static int s_test_known_crc_32(
     uint32_t result = func(input, (int)length, 0);
     ASSERT_HEX_EQUALS(expected_crc, result, "%s(%s)", func_name, data_name);
 
+    uint32_t result_swapped = aws_swap_bytes_if_needed_32(result);
     // Compute the residue of the buffer (the CRC of the buffer plus its CRC) - will always be a constant value
-    uint32_t residue = (uint32_t)func((const uint8_t *)&result, 4, result); // assuming little endian
+    uint32_t residue = (uint32_t)func((const uint8_t *)&result_swapped, 4, result); // assuming little endian
     ASSERT_HEX_EQUALS(expected_residue, residue, "len %d residue %s(%s)", length, func_name, data_name);
 
     // chain the crc computation so 2 calls each operate on about 1/2 of the buffer