From beb9c7d12a577c9953c48c18f39d9100bf6574a3 Mon Sep 17 00:00:00 2001 From: Hisham Mardam-Bey Date: Mon, 8 Jan 2024 18:37:27 -0500 Subject: [PATCH] Added dry-run mode command line option (-n, --dry-run). When invoked, Docuum will report what images would have been deleted during it's initial vacuuming run at start up then exit. In order to do this, Docuum now creates a list of images to delete ensuring that by deleting them it will meet the space requirements set forth by the user. --- README.md | 3 + src/main.rs | 16 +++++ src/run.rs | 173 ++++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 147 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 17840e8..933d085 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,9 @@ OPTIONS: -d, --deletion-chunk-size Removes specified quantity of images at a time (default: 1) + -n, --dry-run + Dry run mode, prevents deletion of any images. + -h, --help Prints help information diff --git a/src/main.rs b/src/main.rs index 7238ab4..0d98a4e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -35,6 +35,7 @@ const DEFAULT_THRESHOLD: &str = "10 GB"; // Command-line argument and option names const DELETION_CHUNK_SIZE_OPTION: &str = "deletion-chunk-size"; const KEEP_OPTION: &str = "keep"; +const DRY_RUN_OPTION: &str = "dry-run"; const THRESHOLD_OPTION: &str = "threshold"; // Size threshold argument, absolute or relative to filesystem size @@ -109,6 +110,7 @@ pub struct Settings { threshold: Threshold, keep: Option, deletion_chunk_size: usize, + dry_run: bool, } // Set up the logger. @@ -186,6 +188,14 @@ fn settings() -> io::Result { .number_of_values(1) .help("Prevents deletion of images for which repository:tag matches "), ) + .arg( + Arg::with_name(DRY_RUN_OPTION) + .short("n") + .long(DRY_RUN_OPTION) + .required(false) + .takes_value(false) + .help("Dry run mode, prevents deletion of any images."), + ) .arg( Arg::with_name(DELETION_CHUNK_SIZE_OPTION) .value_name("DELETION CHUNK SIZE") @@ -216,6 +226,11 @@ fn settings() -> io::Result { None => None, }; + let dry_run = matches.is_present(DRY_RUN_OPTION); + if dry_run { + info!("Dry-run mode enabled, will not be deleting images."); + } + // Determine how many images to delete at once. let deletion_chunk_size = match matches.value_of(DELETION_CHUNK_SIZE_OPTION) { Some(v) => match v.parse::() { @@ -229,6 +244,7 @@ fn settings() -> io::Result { threshold, keep, deletion_chunk_size, + dry_run, }) } diff --git a/src/run.rs b/src/run.rs index badbd87..87cbfce 100644 --- a/src/run.rs +++ b/src/run.rs @@ -4,7 +4,7 @@ use { state::{self, State}, Settings, Threshold, }, - byte_unit::Byte, + byte_unit::{Byte, ByteUnit}, chrono::DateTime, regex::RegexSet, scopeguard::guard, @@ -15,7 +15,7 @@ use { io::{self, BufRead, BufReader}, mem::drop, ops::Deref, - process::{Command, Stdio}, + process::{exit, Child, Command, Stdio}, time::{Duration, SystemTime, UNIX_EPOCH}, }, }; @@ -85,6 +85,7 @@ struct ImageRecord { parent_id: Option, created_since_epoch: Duration, repository_tags: Vec, // [ref:at_least_one_repository_tag] + size: u128, } // This is a node in the image polyforest. Note that the image ID is not included here because this @@ -96,6 +97,15 @@ struct ImageNode { ancestors: usize, // 0 for images with no parent or missing parent } +// Builds an image's name from as repository:tag +fn image_name(image_record: &ImageRecord) -> String { + image_record + .repository_tags + .get(0) + .map(|r| format!("{}:{}", r.repository, r.tag)) + .unwrap() +} + // Ask Docker for the ID of an image. fn image_id(image: &str) -> io::Result { // Query Docker for the image ID. @@ -162,12 +172,12 @@ fn list_image_records(state: &State) -> io::Result> // Get the IDs and creation timestamps of all the images. let output = Command::new("docker") .args([ - "image", - "ls", - "--all", - "--no-trunc", + "system", + "df", + "-v", "--format", - "{{.ID}}\\t{{.Repository}}\\t{{.Tag}}\\t{{.CreatedAt}}", + "{{range .Images}}{{.ID}}\\t{{.Repository}}\\t\ + {{.Tag}}\\t{{.CreatedAt}}\\t{{.UniqueSize}}\n{{end}}", ]) .stderr(Stdio::inherit()) .output()?; @@ -193,13 +203,18 @@ fn list_image_records(state: &State) -> io::Result> } let image_parts = trimmed_line.split('\t').collect::>(); - if let [id, repository, tag, date_str] = image_parts[..] { + if let [sha256_id, repository, tag, date_str, size] = image_parts[..] { let repository_tag = RepositoryTag { repository: repository.to_owned(), tag: tag.to_owned(), }; - match image_records.entry(id.to_owned()) { + // The id is in the format: + // sha256:745895703263072416be27b333d19eff4494b287001f6c6adddd22b963a3429d + // We only want the first 12 characters of the hash. + let id = sha256_id.get(7..19).unwrap(); + + match image_records.entry(id.to_string()) { Entry::Occupied(mut entry) => { (entry.get_mut()).repository_tags.push(repository_tag); } @@ -208,6 +223,7 @@ fn list_image_records(state: &State) -> io::Result> parent_id: parent_id(state, id)?, created_since_epoch: parse_docker_date(date_str)?, repository_tags: vec![repository_tag], + size: parse_docker_image_size(size), }); } } @@ -468,7 +484,13 @@ fn touch_image(state: &mut State, image_id: &str, verbose: bool) -> io::Result u128 { + Byte::from_str(image_size).unwrap().get_bytes() +} + +// Parse the non-standard timestamp format Docker uses for `docker system df -v`. // Example input: "2017-12-20 16:30:49 -0500 EST". fn parse_docker_date(timestamp: &str) -> io::Result { // Chrono can't read the "EST", so remove it before parsing. @@ -629,6 +651,7 @@ fn vacuum( threshold: Byte, keep: &Option, deletion_chunk_size: usize, + dry_run: bool, ) -> io::Result<()> { // Find all images. let image_records = list_image_records(state)?; @@ -681,29 +704,58 @@ fn vacuum( threshold.get_appropriate_unit(false).to_string().code_str(), ); - // Start deleting images, beginning with the least recently used. - for image_ids in sorted_image_nodes.chunks_mut(deletion_chunk_size) { - for (image_id, _) in image_ids { - // Delete the image. - if let Err(error) = delete_image(image_id) { - // The deletion failed. Just log the error and proceed. - error!("{}", error); - } else { - // Forget about the deleted image. - deleted_image_ids.insert(image_id.clone()); + // Create list of images to delete + let mut size_claimed = 0; + let mut images_to_delete = Vec::new(); + 'outer: for image_ids in sorted_image_nodes.chunks_mut(deletion_chunk_size) { + for (image_id, image_node) in image_ids { + info!( + "Marking image for deletion ID: {} name: {} size: {}", + image_id, + image_name(&image_node.image_record), + Byte::from(image_node.image_record.size) + .get_adjusted_unit(ByteUnit::MB) + .to_string(), + ); + size_claimed += image_node.image_record.size; + images_to_delete.push(image_id); + if space.get_bytes() - size_claimed <= threshold.get_bytes() { + break 'outer; } } + } - // Break if we're within the threshold. - let new_space = space_usage()?; - if new_space <= threshold { - info!( - "Docker images are now using {}, which is within the limit of {}.", - new_space.get_appropriate_unit(false).to_string().code_str(), - threshold.get_appropriate_unit(false).to_string().code_str(), - ); - break; + info!( + "We'll claim back {}/{}", + Byte::from(size_claimed) + .get_adjusted_unit(ByteUnit::MB) + .get_value() + .to_string() + .code_str(), + space.get_adjusted_unit(ByteUnit::MB).to_string().code_str(), + ); + + if !dry_run { + // Start deleting images, beginning with the least recently used. + for image_ids in images_to_delete.chunks(deletion_chunk_size) { + for image_id in image_ids { + // Delete the image. + if let Err(error) = delete_image(image_id) { + // The deletion failed. Just log the error and proceed. + error!("{}", error); + } else { + // Forget about the deleted image. + deleted_image_ids.insert((**image_id).to_string()); + } + } } + + let new_space = space_usage()?; + info!( + "Docker images are now using {}, which is within the limit of {}.", + new_space.get_appropriate_unit(false).to_string().code_str(), + threshold.get_appropriate_unit(false).to_string().code_str(), + ); } } else { debug!( @@ -713,10 +765,20 @@ fn vacuum( ); } - // Update the state. + update_state(state, polyforest, &deleted_image_ids); + + Ok(()) +} + +// Update the state. +fn update_state( + state: &mut State, + polyforest: HashMap, + deleted_image_ids: &HashSet, +) { state.images.clear(); for (image_id, image_node) in polyforest { - if !deleted_image_ids.contains(&image_id) { + if !deleted_image_ids.contains(image_id.as_str()) { state.images.insert( image_id.clone(), state::Image { @@ -726,10 +788,7 @@ fn vacuum( ); } } - - Ok(()) } - // Stream Docker events and vacuum when necessary. pub fn run(settings: &Settings, state: &mut State, first_run: &mut bool) -> io::Result<()> { // Determine the threshold in bytes. @@ -759,21 +818,20 @@ pub fn run(settings: &Settings, state: &mut State, first_run: &mut bool) -> io:: threshold, &settings.keep, settings.deletion_chunk_size, + settings.dry_run, )?; state::save(state)?; *first_run = false; - // Spawn `docker events --format '{{json .}}'`. - let mut child = guard( - Command::new("docker") - .args(["events", "--format", "{{json .}}"]) - .stdout(Stdio::piped()) - .spawn()?, - |mut child| { - drop(child.kill()); - drop(child.wait()); - }, - ); + if settings.dry_run { + info!("Exiting now since this is a sample dry-run."); + exit(0); + } + + let mut child = guard(docker_events_listener()?, |mut child| { + drop(child.kill()); + drop(child.wait()); + }); // Buffer the data as we read it line-by-line. let reader = BufReader::new(child.stdout.as_mut().map_or_else( @@ -841,6 +899,7 @@ pub fn run(settings: &Settings, state: &mut State, first_run: &mut bool) -> io:: threshold, &settings.keep, settings.deletion_chunk_size, + settings.dry_run, )?; } @@ -858,6 +917,14 @@ pub fn run(settings: &Settings, state: &mut State, first_run: &mut bool) -> io:: )) } +// Spawn `docker events --format '{{json .}}'`. +fn docker_events_listener() -> io::Result { + Command::new("docker") + .args(["events", "--format", "{{json .}}"]) + .stdout(Stdio::piped()) + .spawn() +} + #[cfg(test)] mod tests { use { @@ -928,6 +995,7 @@ mod tests { repository: String::from("alpine"), tag: String::from("latest"), }], + size: 30000, }; let mut image_records = HashMap::new(); @@ -962,6 +1030,7 @@ mod tests { repository: String::from("alpine"), tag: String::from("latest"), }], + size: 30000, }; let mut image_records = HashMap::new(); @@ -1013,6 +1082,7 @@ mod tests { repository: String::from("alpine"), tag: String::from("latest"), }], + size: 30000, }; let image_record_1 = ImageRecord { @@ -1022,6 +1092,7 @@ mod tests { repository: String::from("debian"), tag: String::from("latest"), }], + size: 200_000, }; let mut image_records = HashMap::new(); @@ -1083,6 +1154,7 @@ mod tests { repository: String::from("alpine"), tag: String::from("latest"), }], + size: 30000, }; let image_record_1 = ImageRecord { @@ -1092,6 +1164,7 @@ mod tests { repository: String::from("debian"), tag: String::from("latest"), }], + size: 200_000, }; let mut image_records = HashMap::new(); @@ -1161,6 +1234,8 @@ mod tests { repository: String::from("alpine"), tag: String::from("latest"), }], + + size: 30000, }; let image_record_1 = ImageRecord { @@ -1170,6 +1245,7 @@ mod tests { repository: String::from("debian"), tag: String::from("latest"), }], + size: 200_000, }; let image_record_2 = ImageRecord { @@ -1179,6 +1255,7 @@ mod tests { repository: String::from("ubuntu"), tag: String::from("latest"), }], + size: 300_000, }; let mut image_records = HashMap::new(); @@ -1258,6 +1335,7 @@ mod tests { repository: String::from("alpine"), tag: String::from("latest"), }], + size: 30000, }; let image_record_1 = ImageRecord { @@ -1267,6 +1345,7 @@ mod tests { repository: String::from("debian"), tag: String::from("latest"), }], + size: 200_000, }; let image_record_2 = ImageRecord { @@ -1276,6 +1355,7 @@ mod tests { repository: String::from("ubuntu"), tag: String::from("latest"), }], + size: 300_000, }; let mut image_records = HashMap::new(); @@ -1355,6 +1435,7 @@ mod tests { repository: String::from("alpine"), tag: String::from("latest"), }], + size: 30000, }; let image_record_1 = ImageRecord { @@ -1364,6 +1445,7 @@ mod tests { repository: String::from("debian"), tag: String::from("latest"), }], + size: 200_000, }; let image_record_2 = ImageRecord { @@ -1373,6 +1455,7 @@ mod tests { repository: String::from("ubuntu"), tag: String::from("latest"), }], + size: 300_000, }; let mut image_records = HashMap::new();