feat: allow compile time override of MAX_CHUNK_SIZE

Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-authors = [ "MaidSafe Developers <[email protected]>" ]
+authors = ["MaidSafe Developers <[email protected]>"]
 description = "Self encrypting files (convergent encryption plus obfuscation)"
 documentation = "https://docs.rs/self_encryption"
 edition = "2018"
@@ -14,6 +14,7 @@ version = "0.29.2"
 aes = "~0.8.1"
 bincode = "~1.3.3"
 hex = "~0.4.3"
+lazy_static = "1.4.0"
 rand = "~0.8.5"
 rand_chacha = "~0.3.1"
 rayon = "1.5.1"
@@ -23,38 +24,38 @@ itertools = "~0.10.0"
 tempfile = "3.6.0"
 xor_name = "5.0.0"
 
-  [dependencies.brotli]
-  version = "~3.3.0"
-  default-features = false
-  features = [ "std" ]
+[dependencies.brotli]
+version = "~3.3.0"
+default-features = false
+features = ["std"]
 
-  [dependencies.cbc]
-  version = "~0.1.1"
-  features = [ "alloc", "block-padding" ]
+[dependencies.cbc]
+version = "~0.1.1"
+features = ["alloc", "block-padding"]
 
-  [dependencies.bytes]
-  version = "1.1.0"
-  features = [ "serde" ]
+[dependencies.bytes]
+version = "1.1.0"
+features = ["serde"]
 
-  [dependencies.serde]
-  version = "1.0.136"
-  features = [ "derive" ]
+[dependencies.serde]
+version = "1.0.136"
+features = ["derive"]
 
-  [dependencies.tiny-keccak]
-  version = "2.0.2"
-  features = [ "sha3" ]
+[dependencies.tiny-keccak]
+version = "2.0.2"
+features = ["sha3"]
 
-  [dependencies.tokio]
-  version = "1.34.0"
-  features = [ "rt" ]
+[dependencies.tokio]
+version = "1.34.0"
+features = ["rt"]
 
 [dev-dependencies]
 criterion = "~0.3"
 docopt = "~0.9.0"
 
-  [dev-dependencies.tokio]
-  version = "1.34.0"
-  features = [ "rt-multi-thread", "macros" ]
+[dev-dependencies.tokio]
+version = "1.34.0"
+features = ["rt-multi-thread", "macros"]
 
 [[example]]
 bench = false

src/lib.rs

@@ -80,7 +80,6 @@
     unused_results
 )]
 #![allow(
-    box_pointers,
     missing_copy_implementations,
     missing_debug_implementations,
     variant_size_differences,
@@ -110,6 +109,7 @@ use chunk::batch_positions;
 use decrypt::decrypt_chunk;
 use encrypt::encrypt_chunk;
 use itertools::Itertools;
+use lazy_static::lazy_static;
 use std::{
     collections::BTreeMap,
     fs::{self, File, OpenOptions},
@@ -126,8 +126,17 @@ pub use xor_name;
 
 /// The minimum size (before compression) of data to be self-encrypted, defined as 3B.
 pub const MIN_ENCRYPTABLE_BYTES: usize = 3 * MIN_CHUNK_SIZE;
-/// The maximum size (before compression) of an individual chunk of a file, defined as 500kiB.
-pub const MAX_CHUNK_SIZE: usize = 512 * 1024;
+/// The default maximum size (before compression) of an individual chunk of a file, defaulting as 1MiB.
+const DEFAULT_MAX_CHUNK_SIZE: usize = 1024 * 1024;
+
+lazy_static! {
+    /// The maximum size (before compression) of an individual chunk of a file, defaulting as 1MiB.
+    pub static ref MAX_CHUNK_SIZE: usize = std::option_env!("MAX_CHUNK_SIZE")
+        .unwrap_or("1048576")
+        .parse::<usize>()
+        .unwrap_or(DEFAULT_MAX_CHUNK_SIZE);
+}
+
 /// The minimum size (before compression) of an individual chunk of a file, defined as 1B.
 pub const MIN_CHUNK_SIZE: usize = 1;
 /// Controls the compression-speed vs compression-density tradeoffs.  The higher the quality, the
@@ -487,7 +496,7 @@ pub struct SeekInfo {
 pub fn seek_info(file_size: usize, pos: usize, len: usize) -> SeekInfo {
     let (start_index, end_index) = overlapped_chunks(file_size, pos, len);
 
-    let relative_pos = if start_index == 2 && file_size < 3 * MAX_CHUNK_SIZE {
+    let relative_pos = if start_index == 2 && file_size < 3 * *MAX_CHUNK_SIZE {
         pos - (2 * get_chunk_size(file_size, 0))
     } else {
         pos % get_chunk_size(file_size, start_index)
@@ -569,13 +578,13 @@ fn get_num_chunks(file_size: usize) -> usize {
     if file_size < (3 * MIN_CHUNK_SIZE) {
         return 0;
     }
-    if file_size < (3 * MAX_CHUNK_SIZE) {
+    if file_size < (3 * *MAX_CHUNK_SIZE) {
         return 3;
     }
-    if file_size % MAX_CHUNK_SIZE == 0 {
-        file_size / MAX_CHUNK_SIZE
+    if file_size % *MAX_CHUNK_SIZE == 0 {
+        file_size / *MAX_CHUNK_SIZE
     } else {
-        (file_size / MAX_CHUNK_SIZE) + 1
+        (file_size / *MAX_CHUNK_SIZE) + 1
     }
 }
 
@@ -584,7 +593,7 @@ fn get_chunk_size(file_size: usize, chunk_index: usize) -> usize {
     if file_size < 3 * MIN_CHUNK_SIZE {
         return 0;
     }
-    if file_size < 3 * MAX_CHUNK_SIZE {
+    if file_size < 3 * *MAX_CHUNK_SIZE {
         if chunk_index < 2 {
             return file_size / 3;
         } else {
@@ -594,21 +603,21 @@ fn get_chunk_size(file_size: usize, chunk_index: usize) -> usize {
     }
     let total_chunks = get_num_chunks(file_size);
     if chunk_index < total_chunks - 2 {
-        return MAX_CHUNK_SIZE;
+        return *MAX_CHUNK_SIZE;
     }
-    let remainder = file_size % MAX_CHUNK_SIZE;
+    let remainder = file_size % *MAX_CHUNK_SIZE;
     let penultimate = (total_chunks - 2) == chunk_index;
     if remainder == 0 {
-        return MAX_CHUNK_SIZE;
+        return *MAX_CHUNK_SIZE;
     }
     if remainder < MIN_CHUNK_SIZE {
         if penultimate {
-            MAX_CHUNK_SIZE - MIN_CHUNK_SIZE
+            *MAX_CHUNK_SIZE - MIN_CHUNK_SIZE
         } else {
             MIN_CHUNK_SIZE + remainder
         }
     } else if penultimate {
-        MAX_CHUNK_SIZE
+        *MAX_CHUNK_SIZE
     } else {
         remainder
     }