From 2e1b019a22e2ef2f9cf79bd15846eaec0fbd6e2a Mon Sep 17 00:00:00 2001 From: Lin Liu Date: Fri, 20 Sep 2024 13:18:10 -0700 Subject: [PATCH] Formatting --- .../action/clean/CleanPlanActionExecutor.java | 26 +++++++++++++------ .../hudi/table/action/clean/CleanPlanner.java | 17 +++++++----- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanActionExecutor.java index 613ed9793678..c2baccf9e6ea 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanActionExecutor.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanActionExecutor.java @@ -114,10 +114,16 @@ HoodieCleanerPlan requestClean(HoodieEngineContext context) { LOG.info("Nothing to clean here. It is already clean"); return HoodieCleanerPlan.newBuilder().setPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS.name()).build(); } - LOG.info("Earliest commit to retain for clean : " + (earliestInstant.isPresent() ? earliestInstant.get().getTimestamp() : "null")); - LOG.info("Total partitions to clean : " + partitionsToClean.size() + ", with policy " + config.getCleanerPolicy()); + LOG.info( + "Earliest commit to retain for clean : {}", + earliestInstant.isPresent() ? earliestInstant.get().getTimestamp() : "null"); + LOG.info( + "Total partitions to clean : {}, with policy {}", + partitionsToClean.size(), + config.getCleanerPolicy()); int cleanerParallelism = Math.min(partitionsToClean.size(), config.getCleanerParallelism()); - LOG.info("Using cleanerParallelism: " + cleanerParallelism); + LOG.info( + "Using cleanerParallelism: {}", cleanerParallelism); context.setJobStatus(this.getClass().getSimpleName(), "Generating list of file slices to be cleaned: " + config.getTableName()); @@ -145,11 +151,15 @@ HoodieCleanerPlan requestClean(HoodieEngineContext context) { .collect(Collectors.toList())); } - return new HoodieCleanerPlan(earliestInstant - .map(x -> new HoodieActionInstant(x.getTimestamp(), x.getAction(), x.getState().name())).orElse(null), - planner.getLastCompletedCommitTimestamp(), - config.getCleanerPolicy().name(), Collections.emptyMap(), - CleanPlanner.LATEST_CLEAN_PLAN_VERSION, cleanOps, partitionsToDelete, prepareExtraMetadata(planner.getSavepointedTimestamps())); + return new HoodieCleanerPlan( + earliestInstant.map(x -> new HoodieActionInstant(x.getTimestamp(), x.getAction(), x.getState().name())).orElse(null), + planner.getLastCompletedCommitTimestamp(), // Note: This is the start time of the last completed ingestion before this clean. + config.getCleanerPolicy().name(), + Collections.emptyMap(), + CleanPlanner.LATEST_CLEAN_PLAN_VERSION, + cleanOps, + partitionsToDelete, + prepareExtraMetadata(planner.getSavepointedTimestamps())); } catch (IOException e) { throw new HoodieIOException("Failed to schedule clean operation", e); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java index 57546252cada..851110ff4610 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java @@ -180,7 +180,7 @@ private List getPartitionPathsForCleanByCommits(Option in HoodieCleanMetadata cleanMetadata = TimelineMetadataUtils .deserializeHoodieCleanMetadata(hoodieTable.getActiveTimeline().getInstantDetails(lastClean.get()).get()); if ((cleanMetadata.getEarliestCommitToRetain() != null) - && (cleanMetadata.getEarliestCommitToRetain().length() > 0) + && !cleanMetadata.getEarliestCommitToRetain().trim().isEmpty() && !hoodieTable.getActiveTimeline().getCommitsTimeline().isBeforeTimelineStarts(cleanMetadata.getEarliestCommitToRetain())) { return getPartitionPathsForIncrementalCleaning(cleanMetadata, instantToRetain); } @@ -204,9 +204,11 @@ private List getPartitionPathsForIncrementalCleaning(HoodieCleanMetadata LOG.info("Since savepoints have been removed compared to previous clean, triggering clean planning for all partitions"); return getPartitionPathsForFullCleaning(); } else { - LOG.info("Incremental Cleaning mode is enabled. Looking up partition-paths that have since changed " - + "since last cleaned at " + cleanMetadata.getEarliestCommitToRetain() - + ". New Instant to retain : " + newInstantToRetain); + LOG.info( + "Incremental Cleaning mode is enabled. Looking up partition-paths that have changed " + + "since last clean at {}. New Instant to retain {}.", + cleanMetadata.getEarliestCommitToRetain(), + newInstantToRetain); return hoodieTable.getCompletedCommitsTimeline().getInstantsAsStream() .filter(instant -> HoodieTimeline.compareTimestamps(instant.getTimestamp(), HoodieTimeline.GREATER_THAN_OR_EQUALS, @@ -288,8 +290,11 @@ private boolean isFileSliceExistInSavepointedFiles(FileSlice fs, List sa * single file (i.e., run it with versionsRetained = 1) */ private Pair> getFilesToCleanKeepingLatestVersions(String partitionPath) { - LOG.info("Cleaning " + partitionPath + ", retaining latest " + config.getCleanerFileVersionsRetained() - + " file versions. "); + LOG.info( + "Cleaning {}, retaining latest {} file versions.", + partitionPath, + config.getCleanerFileVersionsRetained()); + List deletePaths = new ArrayList<>(); // Collect all the datafiles savepointed by all the savepoints List savepointedFiles = hoodieTable.getSavepointTimestamps().stream()