feat: allow compile time override of MAX_CHUNK_SIZE

Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-authors = [ "MaidSafe Developers <[email protected]>" ]
+authors = ["MaidSafe Developers <[email protected]>"]
 description = "Self encrypting files (convergent encryption plus obfuscation)"
 documentation = "https://docs.rs/self_encryption"
 edition = "2018"
@@ -14,47 +14,48 @@ version = "0.29.2"
 aes = "~0.8.1"
 bincode = "~1.3.3"
 hex = "~0.4.3"
+lazy_static = "1.4.0"
 rand = "~0.8.5"
 rand_chacha = "~0.3.1"
 rayon = "1.5.1"
-err-derive = "~0.3.1"
+thiserror = "1.0"
 num_cpus = "1.13.0"
 itertools = "~0.10.0"
 tempfile = "3.6.0"
 xor_name = "5.0.0"
 
-  [dependencies.brotli]
-  version = "~3.3.0"
-  default-features = false
-  features = [ "std" ]
+[dependencies.brotli]
+version = "~3.3.0"
+default-features = false
+features = ["std"]
 
-  [dependencies.cbc]
-  version = "~0.1.1"
-  features = [ "alloc", "block-padding" ]
+[dependencies.cbc]
+version = "~0.1.1"
+features = ["alloc", "block-padding"]
 
-  [dependencies.bytes]
-  version = "1.1.0"
-  features = [ "serde" ]
+[dependencies.bytes]
+version = "1.1.0"
+features = ["serde"]
 
-  [dependencies.serde]
-  version = "1.0.136"
-  features = [ "derive" ]
+[dependencies.serde]
+version = "1.0.136"
+features = ["derive"]
 
-  [dependencies.tiny-keccak]
-  version = "2.0.2"
-  features = [ "sha3" ]
+[dependencies.tiny-keccak]
+version = "2.0.2"
+features = ["sha3"]
 
-  [dependencies.tokio]
-  version = "1.34.0"
-  features = [ "rt" ]
+[dependencies.tokio]
+version = "1.34.0"
+features = ["rt"]
 
 [dev-dependencies]
 criterion = "~0.3"
 docopt = "~0.9.0"
 
-  [dev-dependencies.tokio]
-  version = "1.34.0"
-  features = [ "rt-multi-thread", "macros" ]
+[dev-dependencies.tokio]
+version = "1.34.0"
+features = ["rt-multi-thread", "macros"]
 
 [[example]]
 bench = false

benches/lib.rs

@@ -41,11 +41,7 @@
     unused_qualifications,
     variant_size_differences
 )]
-#![allow(
-    box_pointers,
-    missing_copy_implementations,
-    missing_debug_implementations
-)]
+#![allow(missing_copy_implementations, missing_debug_implementations)]
 
 use criterion::{BatchSize, Bencher, Criterion};
 use self_encryption::{decrypt_full_set, encrypt, test_helpers::random_bytes};

src/chunk.rs

@@ -11,7 +11,6 @@ use bytes::Bytes;
 use rayon::prelude::*;
 use xor_name::XorName;
 
-///
 #[derive(Clone)]
 pub(crate) struct EncryptionBatch {
     pub(crate) raw_chunks: Vec<RawChunk>,

src/error.rs

@@ -7,8 +7,8 @@
 // permissions and limitations relating to use of the SAFE Network Software.
 
 use bincode::ErrorKind;
-use err_derive::Error;
 use std::io::Error as IoError;
+use thiserror::Error;
 
 /// Specialisation of `std::Result` for crate.
 pub type Result<T, E = Error> = std::result::Result<T, E>;
@@ -17,26 +17,26 @@ pub type Result<T, E = Error> = std::result::Result<T, E>;
 #[derive(Debug, Error)]
 #[allow(missing_docs)]
 pub enum Error {
-    #[error(display = "An error during compression or decompression.")]
+    #[error("An error during compression or decompression.")]
     Compression,
-    #[error(display = "An error during initializing CBC-AES cipher instance.")]
+    #[error("An error during initializing CBC-AES cipher instance.")]
     Cipher(String),
-    #[error(display = "An error within the symmetric encryption process.")]
+    #[error("An error within the symmetric encryption process.")]
     Encryption,
-    #[error(display = "An error within the symmetric decryption process({})", _0)]
+    #[error("An error within the symmetric decryption process({})", _0)]
     Decryption(String),
-    #[error(display = "A generic I/O error")]
-    Io(#[source] IoError),
-    #[error(display = "Generic error({})", _0)]
+    #[error("A generic I/O error")]
+    Io(#[from] IoError),
+    #[error("Generic error({})", _0)]
     Generic(String),
-    #[error(display = "Serialisation error")]
-    Bincode(#[source] Box<ErrorKind>),
-    #[error(display = "deserialization")]
+    #[error("Serialisation error")]
+    Bincode(#[from] Box<ErrorKind>),
+    #[error("deserialization")]
     Deserialise,
-    #[error(display = "num parse error")]
-    NumParse(#[source] std::num::ParseIntError),
-    #[error(display = "Rng error")]
-    Rng(#[source] rand::Error),
-    #[error(display = "Unable to obtain lock")]
+    #[error("num parse error")]
+    NumParse(#[from] std::num::ParseIntError),
+    #[error("Rng error")]
+    Rng(#[from] rand::Error),
+    #[error("Unable to obtain lock")]
     Poison,
 }

src/lib.rs

@@ -80,7 +80,6 @@
     unused_results
 )]
 #![allow(
-    box_pointers,
     missing_copy_implementations,
     missing_debug_implementations,
     variant_size_differences,
@@ -110,6 +109,7 @@ use chunk::batch_positions;
 use decrypt::decrypt_chunk;
 use encrypt::encrypt_chunk;
 use itertools::Itertools;
+use lazy_static::lazy_static;
 use std::{
     collections::BTreeMap,
     fs::{self, File, OpenOptions},
@@ -126,8 +126,17 @@ pub use xor_name;
 
 /// The minimum size (before compression) of data to be self-encrypted, defined as 3B.
 pub const MIN_ENCRYPTABLE_BYTES: usize = 3 * MIN_CHUNK_SIZE;
-/// The maximum size (before compression) of an individual chunk of a file, defined as 500kiB.
-pub const MAX_CHUNK_SIZE: usize = 512 * 1024;
+/// The default maximum size (before compression) of an individual chunk of a file, defaulting as 1MiB.
+const DEFAULT_MAX_CHUNK_SIZE: usize = 1024 * 1024;
+
+lazy_static! {
+    /// The maximum size (before compression) of an individual chunk of a file, defaulting as 1MiB.
+    pub static ref MAX_CHUNK_SIZE: usize = std::option_env!("MAX_CHUNK_SIZE")
+        .unwrap_or("1048576")
+        .parse::<usize>()
+        .unwrap_or(DEFAULT_MAX_CHUNK_SIZE);
+}
+
 /// The minimum size (before compression) of an individual chunk of a file, defined as 1B.
 pub const MIN_CHUNK_SIZE: usize = 1;
 /// Controls the compression-speed vs compression-density tradeoffs.  The higher the quality, the
@@ -337,7 +346,7 @@ impl StreamSelfDecryptor {
     // Drain any in-order chunks due to the recent filled in piece.
     fn drain_unprocessed(&mut self) -> Result<()> {
         while let Some(chunk_name) = self.encrypted_chunks.get(&self.chunk_index) {
-            let file_path = self.temp_dir.path().join(&hex::encode(chunk_name));
+            let file_path = self.temp_dir.path().join(hex::encode(chunk_name));
             let mut chunk_file = File::open(file_path)?;
             let mut chunk_data = Vec::new();
             let _ = chunk_file.read_to_end(&mut chunk_data)?;
@@ -367,7 +376,7 @@ pub fn encrypt_from_file(file_path: &Path, output_dir: &Path) -> Result<(DataMap
         let chunk_name = XorName::from_content(&chunk.content);
         chunk_names.push(chunk_name);
 
-        let file_path = output_dir.join(&hex::encode(chunk_name));
+        let file_path = output_dir.join(hex::encode(chunk_name));
         let mut output_file = File::create(file_path)?;
         output_file.write_all(&chunk.content)?;
     }
@@ -385,7 +394,7 @@ pub fn decrypt_from_chunk_files(
     let mut encrypted_chunks = Vec::new();
     for chunk_info in data_map.infos() {
         let chunk_name = chunk_info.dst_hash;
-        let file_path = chunk_dir.join(&hex::encode(chunk_name));
+        let file_path = chunk_dir.join(hex::encode(chunk_name));
         let mut chunk_file = File::open(file_path)?;
         let mut chunk_data = Vec::new();
         let _ = chunk_file.read_to_end(&mut chunk_data)?;
@@ -487,7 +496,7 @@ pub struct SeekInfo {
 pub fn seek_info(file_size: usize, pos: usize, len: usize) -> SeekInfo {
     let (start_index, end_index) = overlapped_chunks(file_size, pos, len);
 
-    let relative_pos = if start_index == 2 && file_size < 3 * MAX_CHUNK_SIZE {
+    let relative_pos = if start_index == 2 && file_size < 3 * *MAX_CHUNK_SIZE {
         pos - (2 * get_chunk_size(file_size, 0))
     } else {
         pos % get_chunk_size(file_size, start_index)
@@ -569,13 +578,13 @@ fn get_num_chunks(file_size: usize) -> usize {
     if file_size < (3 * MIN_CHUNK_SIZE) {
         return 0;
     }
-    if file_size < (3 * MAX_CHUNK_SIZE) {
+    if file_size < (3 * *MAX_CHUNK_SIZE) {
         return 3;
     }
-    if file_size % MAX_CHUNK_SIZE == 0 {
-        file_size / MAX_CHUNK_SIZE
+    if file_size % *MAX_CHUNK_SIZE == 0 {
+        file_size / *MAX_CHUNK_SIZE
     } else {
-        (file_size / MAX_CHUNK_SIZE) + 1
+        (file_size / *MAX_CHUNK_SIZE) + 1
     }
 }
 
@@ -584,7 +593,7 @@ fn get_chunk_size(file_size: usize, chunk_index: usize) -> usize {
     if file_size < 3 * MIN_CHUNK_SIZE {
         return 0;
     }
-    if file_size < 3 * MAX_CHUNK_SIZE {
+    if file_size < 3 * *MAX_CHUNK_SIZE {
         if chunk_index < 2 {
             return file_size / 3;
         } else {
@@ -594,21 +603,21 @@ fn get_chunk_size(file_size: usize, chunk_index: usize) -> usize {
     }
     let total_chunks = get_num_chunks(file_size);
     if chunk_index < total_chunks - 2 {
-        return MAX_CHUNK_SIZE;
+        return *MAX_CHUNK_SIZE;
     }
-    let remainder = file_size % MAX_CHUNK_SIZE;
+    let remainder = file_size % *MAX_CHUNK_SIZE;
     let penultimate = (total_chunks - 2) == chunk_index;
     if remainder == 0 {
-        return MAX_CHUNK_SIZE;
+        return *MAX_CHUNK_SIZE;
     }
     if remainder < MIN_CHUNK_SIZE {
         if penultimate {
-            MAX_CHUNK_SIZE - MIN_CHUNK_SIZE
+            *MAX_CHUNK_SIZE - MIN_CHUNK_SIZE
         } else {
             MIN_CHUNK_SIZE + remainder
         }
     } else if penultimate {
-        MAX_CHUNK_SIZE
+        *MAX_CHUNK_SIZE
     } else {
         remainder
     }

src/tests.rs

@@ -82,7 +82,7 @@ fn test_stream_self_encryptor() -> Result<(), Error> {
     // Use the flushed encrypted chunks to recover the file and verify with the original data
     let mut flushed_encrypted_chunks = Vec::new();
     for chunk_info in data_map.infos() {
-        let file_path = chunk_path.join(&hex::encode(chunk_info.dst_hash));
+        let file_path = chunk_path.join(hex::encode(chunk_info.dst_hash));
         let mut chunk_file = File::open(file_path)?;
         let mut chunk_data = Vec::new();
         let _ = chunk_file.read_to_end(&mut chunk_data)?;