diff --git a/application/src/main/java/bisq/application/ApplicationService.java b/application/src/main/java/bisq/application/ApplicationService.java index 7bbf3a0f80..e03c57a325 100644 --- a/application/src/main/java/bisq/application/ApplicationService.java +++ b/application/src/main/java/bisq/application/ApplicationService.java @@ -203,6 +203,10 @@ private void checkInstanceLock() { } } + public CompletableFuture pruneAllBackups() { + return persistenceService.pruneAllBackups(); + } + public CompletableFuture readAllPersisted() { return persistenceService.readAllPersisted(); } diff --git a/application/src/main/java/bisq/application/Executable.java b/application/src/main/java/bisq/application/Executable.java index cc5949e7bd..fd178d7bfe 100644 --- a/application/src/main/java/bisq/application/Executable.java +++ b/application/src/main/java/bisq/application/Executable.java @@ -27,7 +27,15 @@ public Executable(String[] args) { })); applicationService = createApplicationService(args); + + long ts = System.currentTimeMillis(); + applicationService.pruneAllBackups().join(); + log.info("pruneAllBackups took {} ms", System.currentTimeMillis() - ts); + + ts = System.currentTimeMillis(); applicationService.readAllPersisted().join(); + log.info("readAllPersisted took {} ms", System.currentTimeMillis() - ts); + launchApplication(args); } diff --git a/apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/SettingsController.java b/apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/SettingsController.java index 374f42acd9..aee9a58888 100644 --- a/apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/SettingsController.java +++ b/apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/SettingsController.java @@ -23,7 +23,7 @@ import bisq.desktop.main.content.ContentTabController; import bisq.desktop.main.content.settings.display.DisplaySettingsController; import bisq.desktop.main.content.settings.language.LanguageSettingsController; -import bisq.desktop.main.content.settings.network.NetworkSettingsController; +import bisq.desktop.main.content.settings.misc.MiscSettingsController; import bisq.desktop.main.content.settings.notifications.NotificationsSettingsController; import bisq.desktop.main.content.settings.trade.TradeSettingsController; import lombok.Getter; @@ -48,7 +48,7 @@ protected Optional createController(NavigationTarget navig case NOTIFICATION_SETTINGS -> Optional.of(new NotificationsSettingsController(serviceProvider)); case DISPLAY_SETTINGS -> Optional.of(new DisplaySettingsController(serviceProvider)); case TRADE_SETTINGS -> Optional.of(new TradeSettingsController(serviceProvider)); - case NETWORK_SETTINGS -> Optional.of(new NetworkSettingsController(serviceProvider)); + case MISC_SETTINGS -> Optional.of(new MiscSettingsController(serviceProvider)); default -> Optional.empty(); }; } diff --git a/apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/SettingsView.java b/apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/SettingsView.java index 7da111938d..1bd93d9127 100644 --- a/apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/SettingsView.java +++ b/apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/SettingsView.java @@ -29,6 +29,6 @@ public SettingsView(SettingsModel model, SettingsController controller) { addTab(Res.get("settings.notifications"), NavigationTarget.NOTIFICATION_SETTINGS); addTab(Res.get("settings.trade"), NavigationTarget.TRADE_SETTINGS); addTab(Res.get("settings.display"), NavigationTarget.DISPLAY_SETTINGS); - addTab(Res.get("settings.network"), NavigationTarget.NETWORK_SETTINGS); + addTab(Res.get("settings.misc"), NavigationTarget.MISC_SETTINGS); } } diff --git a/apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/network/NetworkSettingsController.java b/apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/misc/MiscSettingsController.java similarity index 86% rename from apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/network/NetworkSettingsController.java rename to apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/misc/MiscSettingsController.java index 7d24f3327b..f947dfd180 100644 --- a/apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/network/NetworkSettingsController.java +++ b/apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/misc/MiscSettingsController.java @@ -15,7 +15,7 @@ * along with Bisq. If not, see . */ -package bisq.desktop.main.content.settings.network; +package bisq.desktop.main.content.settings.misc; import bisq.bonded_roles.security_manager.difficulty_adjustment.DifficultyAdjustmentService; import bisq.common.observable.Pin; @@ -31,22 +31,22 @@ import org.fxmisc.easybind.Subscription; @Slf4j -public class NetworkSettingsController implements Controller { +public class MiscSettingsController implements Controller { @Getter - private final NetworkSettingsView view; - private final NetworkSettingsModel model; + private final MiscSettingsView view; + private final MiscSettingsModel model; private final SettingsService settingsService; private final DifficultyAdjustmentService difficultyAdjustmentService; private Pin ignoreDiffAdjustmentFromSecManagerPin, - mostRecentDifficultyAdjustmentFactorOrDefaultPin, difficultyAdjustmentFactorPin; + mostRecentDifficultyAdjustmentFactorOrDefaultPin, difficultyAdjustmentFactorPin, totalMaxBackupSizeInMBPin; private Subscription difficultyAdjustmentFactorDescriptionTextPin; - public NetworkSettingsController(ServiceProvider serviceProvider) { + public MiscSettingsController(ServiceProvider serviceProvider) { settingsService = serviceProvider.getSettingsService(); difficultyAdjustmentService = serviceProvider.getBondedRolesService().getDifficultyAdjustmentService(); - model = new NetworkSettingsModel(); - view = new NetworkSettingsView(model, this); + model = new MiscSettingsModel(); + view = new MiscSettingsView(model, this); } @Override @@ -74,6 +74,9 @@ public void onActivate() { UIThread.run(() -> model.getDifficultyAdjustmentFactor().set(mostRecentValueOrDefault))); } }); + + totalMaxBackupSizeInMBPin = FxBindings.bindBiDir(model.getTotalMaxBackupSizeInMB()) + .to(settingsService.getTotalMaxBackupSizeInMB()); } @Override @@ -81,6 +84,7 @@ public void onDeactivate() { ignoreDiffAdjustmentFromSecManagerPin.unbind(); model.getDifficultyAdjustmentFactorEditable().unbind(); difficultyAdjustmentFactorDescriptionTextPin.unsubscribe(); + totalMaxBackupSizeInMBPin.unbind(); if (difficultyAdjustmentFactorPin != null) { difficultyAdjustmentFactorPin.unbind(); } diff --git a/apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/network/NetworkSettingsModel.java b/apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/misc/MiscSettingsModel.java similarity index 84% rename from apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/network/NetworkSettingsModel.java rename to apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/misc/MiscSettingsModel.java index aa22e52c0d..cf3afd8069 100644 --- a/apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/network/NetworkSettingsModel.java +++ b/apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/misc/MiscSettingsModel.java @@ -15,7 +15,7 @@ * along with Bisq. If not, see . */ -package bisq.desktop.main.content.settings.network; +package bisq.desktop.main.content.settings.misc; import bisq.desktop.common.view.Model; import javafx.beans.property.*; @@ -24,12 +24,13 @@ @Slf4j @Getter -public class NetworkSettingsModel implements Model { +public class MiscSettingsModel implements Model { private final DoubleProperty difficultyAdjustmentFactor = new SimpleDoubleProperty(); private final BooleanProperty difficultyAdjustmentFactorEditable = new SimpleBooleanProperty(); private final StringProperty difficultyAdjustmentFactorDescriptionText = new SimpleStringProperty(); private final BooleanProperty ignoreDiffAdjustmentFromSecManager = new SimpleBooleanProperty(); + private final DoubleProperty totalMaxBackupSizeInMB = new SimpleDoubleProperty(); - public NetworkSettingsModel() { + public MiscSettingsModel() { } } diff --git a/apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/network/NetworkSettingsView.java b/apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/misc/MiscSettingsView.java similarity index 65% rename from apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/network/NetworkSettingsView.java rename to apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/misc/MiscSettingsView.java index ba3531dd4d..9c70318deb 100644 --- a/apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/network/NetworkSettingsView.java +++ b/apps/desktop/desktop/src/main/java/bisq/desktop/main/content/settings/misc/MiscSettingsView.java @@ -15,9 +15,11 @@ * along with Bisq. If not, see . */ -package bisq.desktop.main.content.settings.network; +package bisq.desktop.main.content.settings.misc; +import bisq.common.util.MathUtils; import bisq.desktop.common.converters.Converters; +import bisq.desktop.common.converters.DoubleStringConverter; import bisq.desktop.common.view.View; import bisq.desktop.components.controls.MaterialTextField; import bisq.desktop.components.controls.Switch; @@ -26,6 +28,8 @@ import bisq.desktop.main.content.settings.SettingsViewUtils; import bisq.i18n.Res; import bisq.network.p2p.node.network_load.NetworkLoad; +import bisq.persistence.backup.BackupService; +import de.jensd.fx.fontawesome.AwesomeIcon; import javafx.beans.binding.Bindings; import javafx.geometry.Insets; import javafx.geometry.Pos; @@ -36,17 +40,20 @@ import org.fxmisc.easybind.Subscription; @Slf4j -public class NetworkSettingsView extends View { +public class MiscSettingsView extends View { + private static final double TEXT_FIELD_WIDTH = 500; private static final ValidatorBase DIFFICULTY_ADJUSTMENT_FACTOR_VALIDATOR = new NumberValidator(Res.get("settings.network.difficultyAdjustmentFactor.invalid", NetworkLoad.MAX_DIFFICULTY_ADJUSTMENT), 0, NetworkLoad.MAX_DIFFICULTY_ADJUSTMENT); - private static final double TEXT_FIELD_WIDTH = 500; + private static final ValidatorBase TOTAL_MAX_BACKUP_SIZE_VALIDATOR = + new NumberValidator(Res.get("settings.backup.totalMaxBackupSizeInMB.invalid", 1, 1000), + 1, 1000); private final Switch ignoreDiffAdjustFromSecManagerSwitch; - private final MaterialTextField difficultyAdjustmentFactor; + private final MaterialTextField difficultyAdjustmentFactor, totalMaxBackupSizeInMB; private Subscription ignoreDiffAdjustFromSecManagerSwitchPin; - public NetworkSettingsView(NetworkSettingsModel model, NetworkSettingsController controller) { + public MiscSettingsView(MiscSettingsModel model, MiscSettingsController controller) { super(new VBox(), model, controller); root.setAlignment(Pos.TOP_LEFT); @@ -61,8 +68,20 @@ public NetworkSettingsView(NetworkSettingsModel model, NetworkSettingsController ignoreDiffAdjustFromSecManagerSwitch = new Switch(Res.get("settings.network.difficultyAdjustmentFactor.ignoreValueFromSecManager")); VBox networkVBox = new VBox(10, difficultyAdjustmentFactor, ignoreDiffAdjustFromSecManagerSwitch); + + Label backupHeadline = new Label(Res.get("settings.backup.headline")); + backupHeadline.getStyleClass().add("large-thin-headline"); + + totalMaxBackupSizeInMB = new MaterialTextField(Res.get("settings.backup.totalMaxBackupSizeInMB.description")); + totalMaxBackupSizeInMB.setMaxWidth(TEXT_FIELD_WIDTH); + totalMaxBackupSizeInMB.setValidators(TOTAL_MAX_BACKUP_SIZE_VALIDATOR); + totalMaxBackupSizeInMB.setStringConverter(Converters.DOUBLE_STRING_CONVERTER); + totalMaxBackupSizeInMB.setIcon(AwesomeIcon.INFO_SIGN); + totalMaxBackupSizeInMB.setIconTooltip(Res.get("settings.backup.totalMaxBackupSizeInMB.info.tooltip")); + VBox contentBox = new VBox(50); - contentBox.getChildren().addAll(networkHeadline, SettingsViewUtils.getLineAfterHeadline(contentBox.getSpacing()), networkVBox); + contentBox.getChildren().addAll(networkHeadline, SettingsViewUtils.getLineAfterHeadline(contentBox.getSpacing()), networkVBox, + backupHeadline, SettingsViewUtils.getLineAfterHeadline(contentBox.getSpacing()), totalMaxBackupSizeInMB); contentBox.getStyleClass().add("bisq-common-bg"); root.getChildren().add(contentBox); root.setPadding(new Insets(0, 40, 20, 40)); @@ -75,6 +94,20 @@ protected void onViewAttached() { Converters.DOUBLE_STRING_CONVERTER); difficultyAdjustmentFactor.getTextInputControl().editableProperty().bind(model.getDifficultyAdjustmentFactorEditable()); difficultyAdjustmentFactor.descriptionProperty().bind(model.getDifficultyAdjustmentFactorDescriptionText()); + + Bindings.bindBidirectional(totalMaxBackupSizeInMB.textProperty(), model.getTotalMaxBackupSizeInMB(), + new DoubleStringConverter() { + @Override + public Number fromString(String value) { + double result = MathUtils.parseToDouble(value); + if(TOTAL_MAX_BACKUP_SIZE_VALIDATOR.validateAndGet()){ + return result; + }else{ + return BackupService.TOTAL_MAX_BACKUP_SIZE_IN_MB; + } + } + }); + ignoreDiffAdjustFromSecManagerSwitchPin = EasyBind.subscribe( ignoreDiffAdjustFromSecManagerSwitch.selectedProperty(), s -> difficultyAdjustmentFactor.validate()); } @@ -85,7 +118,11 @@ protected void onViewDetached() { Bindings.unbindBidirectional(difficultyAdjustmentFactor.textProperty(), model.getDifficultyAdjustmentFactor()); difficultyAdjustmentFactor.getTextInputControl().editableProperty().unbind(); difficultyAdjustmentFactor.descriptionProperty().unbind(); + + Bindings.unbindBidirectional(totalMaxBackupSizeInMB.textProperty(), model.getTotalMaxBackupSizeInMB()); + ignoreDiffAdjustFromSecManagerSwitchPin.unsubscribe(); difficultyAdjustmentFactor.resetValidation(); + totalMaxBackupSizeInMB.resetValidation(); } } diff --git a/bisq-easy/src/main/java/bisq/bisq_easy/NavigationTarget.java b/bisq-easy/src/main/java/bisq/bisq_easy/NavigationTarget.java index 263fbf2aeb..3a6f93196d 100644 --- a/bisq-easy/src/main/java/bisq/bisq_easy/NavigationTarget.java +++ b/bisq-easy/src/main/java/bisq/bisq_easy/NavigationTarget.java @@ -195,7 +195,7 @@ public enum NavigationTarget { NOTIFICATION_SETTINGS(SETTINGS), DISPLAY_SETTINGS(SETTINGS), TRADE_SETTINGS(SETTINGS), - NETWORK_SETTINGS(SETTINGS), + MISC_SETTINGS(SETTINGS), // WALLET WALLET(CONTENT), diff --git a/common/src/main/java/bisq/common/file/FileUtils.java b/common/src/main/java/bisq/common/file/FileUtils.java index 8956e471bf..37da0d4eca 100644 --- a/common/src/main/java/bisq/common/file/FileUtils.java +++ b/common/src/main/java/bisq/common/file/FileUtils.java @@ -164,6 +164,10 @@ public FileVisitResult postVisitDirectory(Path dir, IOException e) })); } + public static void makeDirs(Path dirPath) throws IOException { + makeDirs(dirPath.toFile()); + } + public static void makeDirs(String dirPath) throws IOException { makeDirs(new File(dirPath)); } @@ -288,7 +292,14 @@ public static void copy(InputStream inputStream, OutputStream outputStream) thro } public static Set listFiles(String dirPath) { - try (Stream stream = Files.list(Paths.get(dirPath))) { + return listFiles(Paths.get(dirPath)); + } + + public static Set listFiles(Path dirPath) { + if (!dirPath.toFile().exists()) { + return new HashSet<>(); + } + try (Stream stream = Files.list(dirPath)) { return stream .filter(file -> !Files.isDirectory(file)) .map(Path::getFileName) @@ -300,6 +311,26 @@ public static Set listFiles(String dirPath) { } } + public static Set listDirectories(String dirPath) { + return listDirectories(Paths.get(dirPath)); + } + + public static Set listDirectories(Path dirPath) { + if (!dirPath.toFile().exists()) { + return new HashSet<>(); + } + try (Stream stream = Files.list(dirPath)) { + return stream + .filter(Files::isDirectory) + .map(Path::getFileName) + .map(Path::toString) + .collect(Collectors.toSet()); + } catch (IOException e) { + log.error(e.toString(), e); + return new HashSet<>(); + } + } + public static File createNewFile(Path path) throws IOException { File file = path.toFile(); if (!file.createNewFile()) { diff --git a/i18n/src/main/resources/settings.properties b/i18n/src/main/resources/settings.properties index 53d4608dc9..d2c711a372 100644 --- a/i18n/src/main/resources/settings.properties +++ b/i18n/src/main/resources/settings.properties @@ -6,7 +6,7 @@ settings.language=Language settings.notifications=Notifications settings.trade=Offer and trade settings.display=Display -settings.network=Network +settings.misc=Miscellaneous settings.language.headline=Language selection settings.language.select=Select language @@ -44,3 +44,16 @@ settings.network.difficultyAdjustmentFactor.description.fromSecManager=PoW diffi settings.network.difficultyAdjustmentFactor.invalid=Must be a number between 0 and {0} settings.network.difficultyAdjustmentFactor.ignoreValueFromSecManager=Ignore value provided by Bisq Security Manager +settings.backup.headline=Backup settings +settings.backup.totalMaxBackupSizeInMB.description=Max. size in MB for automatic backups +settings.backup.totalMaxBackupSizeInMB.info.tooltip=Important data is automatically backed up in the data directory whenever updates are made,\n\ + following this retention strategy:\n\t\ + - Last Hour: Keep a maximum of one backup per minute.\n\t\ + - Last Day: Keep one backup per hour.\n\t\ + - Last Week: Keep one backup per day.\n\t\ + - Last Month: Keep one backup per week.\n\t\ + - Last Year: Keep one backup per month.\n\t\ + - Previous Years: Keep one backup per year.\n\n\ + Please note, this backup mechanism does not protect against hard disk failures.\n\ + For better protection, we strongly recommend creating manual backups on an alternative hard drive. +settings.backup.totalMaxBackupSizeInMB.invalid=Must be a value between {0} MB and {1} MB diff --git a/i18n/src/main/resources/settings_cs.properties b/i18n/src/main/resources/settings_cs.properties index 5c62a37ed7..a91110fc4c 100644 --- a/i18n/src/main/resources/settings_cs.properties +++ b/i18n/src/main/resources/settings_cs.properties @@ -6,7 +6,7 @@ settings.language=Jazyk settings.notifications=Oznámení settings.trade=Nabídka a obchod settings.display=Zobrazení -settings.network=Síť +settings.misc=Různé settings.language.headline=Výběr jazyka settings.language.select=Vyberte jazyk diff --git a/i18n/src/main/resources/settings_de.properties b/i18n/src/main/resources/settings_de.properties index 5e182c0795..96d6a7dcc1 100644 --- a/i18n/src/main/resources/settings_de.properties +++ b/i18n/src/main/resources/settings_de.properties @@ -6,7 +6,7 @@ settings.language=Sprache settings.notifications=Benachrichtigungen settings.trade=Angebot und Handel settings.display=Anzeige -settings.network=Netzwerk +settings.misc=Verschiedenes settings.language.headline=Sprachauswahl settings.language.select=Sprache auswählen diff --git a/i18n/src/main/resources/settings_es.properties b/i18n/src/main/resources/settings_es.properties index 89798153e7..48bce2b9b5 100644 --- a/i18n/src/main/resources/settings_es.properties +++ b/i18n/src/main/resources/settings_es.properties @@ -6,7 +6,7 @@ settings.language=Idioma settings.notifications=Notificaciones settings.trade=Oferta y compra-venta settings.display=Visualización -settings.network=Red +settings.misc=Misceláneo settings.language.headline=Selección de Idioma settings.language.select=Seleccionar idioma diff --git a/i18n/src/main/resources/settings_it.properties b/i18n/src/main/resources/settings_it.properties index 905486a70c..762dbcd509 100644 --- a/i18n/src/main/resources/settings_it.properties +++ b/i18n/src/main/resources/settings_it.properties @@ -6,7 +6,7 @@ settings.language=Lingua settings.notifications=Notifiche settings.trade=Offerta e scambio settings.display=Visualizzazione -settings.network=Rete +settings.misc=Misto settings.language.headline=Selezione della lingua settings.language.select=Seleziona la lingua diff --git a/i18n/src/main/resources/settings_pcm.properties b/i18n/src/main/resources/settings_pcm.properties index 9b151fc476..a46771419c 100644 --- a/i18n/src/main/resources/settings_pcm.properties +++ b/i18n/src/main/resources/settings_pcm.properties @@ -6,7 +6,7 @@ settings.language=Language settings.notifications=Notifications settings.trade=Offer and trade settings.display=Display -settings.network=Network +settings.misc=Miscellaneous settings.language.headline=Selek Language settings.language.select=Chuze Language diff --git a/i18n/src/main/resources/settings_pt_BR.properties b/i18n/src/main/resources/settings_pt_BR.properties index fc0b9de93d..d457f06e12 100644 --- a/i18n/src/main/resources/settings_pt_BR.properties +++ b/i18n/src/main/resources/settings_pt_BR.properties @@ -6,7 +6,7 @@ settings.language=Idioma settings.notifications=Notificações settings.trade=Oferta e negociação settings.display=Exibição -settings.network=Rede +settings.misc=Diversos settings.language.headline=Seleção de Idioma settings.language.select=Selecionar idioma diff --git a/identity/src/main/java/bisq/identity/IdentityService.java b/identity/src/main/java/bisq/identity/IdentityService.java index 28f5d10ff2..96923a02c9 100644 --- a/identity/src/main/java/bisq/identity/IdentityService.java +++ b/identity/src/main/java/bisq/identity/IdentityService.java @@ -115,8 +115,7 @@ public CompletableFuture shutdown() { @Override public CompletableFuture persist() { - return getPersistence().persistAsync(getPersistableStore().getClone()) - .handle((nil, throwable) -> throwable == null); + return PersistenceClient.super.persist(); } diff --git a/network/network/src/main/java/bisq/network/p2p/services/data/storage/DataStorageService.java b/network/network/src/main/java/bisq/network/p2p/services/data/storage/DataStorageService.java index ea0966e0e3..763c42fff4 100644 --- a/network/network/src/main/java/bisq/network/p2p/services/data/storage/DataStorageService.java +++ b/network/network/src/main/java/bisq/network/p2p/services/data/storage/DataStorageService.java @@ -30,6 +30,7 @@ import bisq.persistence.Persistence; import bisq.persistence.PersistenceService; import bisq.persistence.RateLimitedPersistenceClient; +import bisq.persistence.backup.MaxBackupSize; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -59,11 +60,13 @@ public DataStorageService(PersistenceService persistenceService, String storeNam this.storeKey = storeKey; String storageFileName = storeKey + STORE_POST_FIX; - subDirectory = DbSubDirectory.NETWORK_DB.getDbPath() + File.separator + storeName; + DbSubDirectory dbSubDirectory = DbSubDirectory.NETWORK_DB; + subDirectory = dbSubDirectory.getDbPath() + File.separator + storeName; persistence = persistenceService.getOrCreatePersistence(this, subDirectory, storageFileName, - persistableStore); + persistableStore, + MaxBackupSize.from(dbSubDirectory)); } public void shutdown() { diff --git a/persistence/src/main/java/bisq/persistence/PersistableStoreFileManager.java b/persistence/src/main/java/bisq/persistence/PersistableStoreFileManager.java index bd80f564b0..e2d35e592d 100644 --- a/persistence/src/main/java/bisq/persistence/PersistableStoreFileManager.java +++ b/persistence/src/main/java/bisq/persistence/PersistableStoreFileManager.java @@ -17,34 +17,37 @@ package bisq.persistence; +import bisq.persistence.backup.BackupService; +import bisq.persistence.backup.MaxBackupSize; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import java.io.File; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; @Slf4j public class PersistableStoreFileManager { - - public static final String BACKUP_DIR = "backup"; public static final String TEMP_FILE_PREFIX = "temp_"; @Getter private final Path storeFilePath; private final Path parentDirectoryPath; - - private final Path backupFilePath; + private final BackupService backupService; @Getter private final Path tempFilePath; public PersistableStoreFileManager(Path storeFilePath) { + this(storeFilePath, MaxBackupSize.ZERO); + } + + public PersistableStoreFileManager(Path storeFilePath, MaxBackupSize maxBackupSize) { this.storeFilePath = storeFilePath; this.parentDirectoryPath = storeFilePath.getParent(); - this.backupFilePath = createBackupFilePath(); this.tempFilePath = createTempFilePath(); + Path dataDir = storeFilePath.getParent().getParent().getParent(); + backupService = new BackupService(dataDir, storeFilePath, maxBackupSize); } public void createParentDirectoriesIfNotExisting() { @@ -57,33 +60,8 @@ public void createParentDirectoriesIfNotExisting() { } } - public void tryToBackupCurrentStoreFile() throws IOException { - File storeFile = storeFilePath.toFile(); - if (!storeFile.exists()) { - return; - } - - File backupFile = backupFilePath.toFile(); - if (backupFile.exists()) { - Files.delete(backupFilePath); - } - - boolean isSuccess = storeFilePath.toFile().renameTo(backupFile); - if (!isSuccess) { - throw new IOException("Couldn't rename " + storeFilePath + " to " + backupFilePath); - } - } - - public void restoreBackupFileIfCurrentFileNotExisting() { - File storeFile = storeFilePath.toFile(); - if (!storeFile.exists()) { - File backupFile = backupFilePath.toFile(); - boolean isSuccess = backupFile.renameTo(storeFile); - - if (!isSuccess) { - log.error("Couldn't rename {} to {}", backupFile, storeFilePath); - } - } + public void maybeMigrateLegacyBackupFile() { + backupService.maybeMigrateLegacyBackupFile(); } public void renameTempFileToCurrentFile() throws IOException { @@ -103,10 +81,12 @@ public void renameTempFileToCurrentFile() throws IOException { } } - private Path createBackupFilePath() { - Path dirPath = Path.of(parentDirectoryPath.toString(), BACKUP_DIR); - dirPath.toFile().mkdirs(); - return dirPath.resolve(storeFilePath.getFileName()); + public boolean maybeBackup() { + return backupService.maybeBackup(); + } + + public void pruneBackups() { + backupService.prune(); } private Path createTempFilePath() { diff --git a/persistence/src/main/java/bisq/persistence/PersistableStoreReaderWriter.java b/persistence/src/main/java/bisq/persistence/PersistableStoreReaderWriter.java index ae8a4cf709..b701201964 100644 --- a/persistence/src/main/java/bisq/persistence/PersistableStoreReaderWriter.java +++ b/persistence/src/main/java/bisq/persistence/PersistableStoreReaderWriter.java @@ -47,6 +47,11 @@ public synchronized Optional read() { return Optional.empty(); } + // In case we do not have any backup file yet, we check if we have a legacy backup file (pre-v2.1.2) and move + // that to the new backup structure. As we only do the backup at write we would otherwise not have data which + // have been written only once like the user identity. + storeFileManager.maybeMigrateLegacyBackupFile(); + try { PersistableStore persistableStore = readStoreFromFile(); //noinspection unchecked,rawtypes @@ -62,21 +67,25 @@ public synchronized Optional read() { public synchronized void write(T persistableStore) { storeFileManager.createParentDirectoriesIfNotExisting(); - try { writeStoreToTempFile(persistableStore); - storeFileManager.tryToBackupCurrentStoreFile(); + boolean hasFileBeenBackedUp = storeFileManager.maybeBackup(); + if (!hasFileBeenBackedUp) { + File storeFile = storeFilePath.toFile(); + FileUtils.deleteFile(storeFile); + } storeFileManager.renameTempFileToCurrentFile(); - } catch (CouldNotSerializePersistableStore e) { log.error("Couldn't serialize {}", persistableStore, e); - } catch (Exception e) { - log.error("Couldn't write persistable store to disk. Trying restore backup.", e); - storeFileManager.restoreBackupFileIfCurrentFileNotExisting(); + log.error("Couldn't write persistable store to disk.", e); } } + public void pruneBackups() { + storeFileManager.pruneBackups(); + } + private PersistableStore readStoreFromFile() throws IOException { File storeFile = storeFilePath.toFile(); try (FileInputStream fileInputStream = new FileInputStream(storeFile)) { diff --git a/persistence/src/main/java/bisq/persistence/Persistence.java b/persistence/src/main/java/bisq/persistence/Persistence.java index 8e60c33e55..bd7a903b3a 100644 --- a/persistence/src/main/java/bisq/persistence/Persistence.java +++ b/persistence/src/main/java/bisq/persistence/Persistence.java @@ -19,6 +19,7 @@ import bisq.common.threading.ExecutorFactory; import bisq.common.util.StringUtils; +import bisq.persistence.backup.MaxBackupSize; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -40,11 +41,11 @@ public class Persistence> { private final PersistableStoreReaderWriter persistableStoreReaderWriter; - public Persistence(String directory, String fileName) { + public Persistence(String directory, String fileName, MaxBackupSize maxBackupSize) { this.fileName = fileName; String storageFileName = StringUtils.camelCaseToSnakeCase(fileName); storePath = Path.of(directory, storageFileName + EXTENSION); - var storeFileManager = new PersistableStoreFileManager(storePath); + var storeFileManager = new PersistableStoreFileManager(storePath, maxBackupSize); persistableStoreReaderWriter = new PersistableStoreReaderWriter<>(storeFileManager); } @@ -63,4 +64,8 @@ public CompletableFuture persistAsync(T serializable) { protected void persist(T persistableStore) { persistableStoreReaderWriter.write(persistableStore); } + + public CompletableFuture pruneBackups() { + return CompletableFuture.runAsync(persistableStoreReaderWriter::pruneBackups, executorService); + } } diff --git a/persistence/src/main/java/bisq/persistence/PersistenceService.java b/persistence/src/main/java/bisq/persistence/PersistenceService.java index e0ab2ff9c8..67d86c064d 100644 --- a/persistence/src/main/java/bisq/persistence/PersistenceService.java +++ b/persistence/src/main/java/bisq/persistence/PersistenceService.java @@ -19,6 +19,7 @@ import bisq.common.proto.PersistableProto; import bisq.common.util.CompletableFutureUtils; +import bisq.persistence.backup.MaxBackupSize; import com.google.common.base.Joiner; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -44,27 +45,66 @@ public PersistenceService(String baseDir) { public > Persistence getOrCreatePersistence(PersistenceClient client, DbSubDirectory dbSubDirectory, PersistableStore persistableStore) { - return getOrCreatePersistence(client, dbSubDirectory.getDbPath(), persistableStore.getClass().getSimpleName(), persistableStore); + return getOrCreatePersistence(client, + dbSubDirectory, + persistableStore.getClass().getSimpleName(), + persistableStore); } public > Persistence getOrCreatePersistence(PersistenceClient client, DbSubDirectory dbSubDirectory, String fileName, PersistableStore persistableStore) { - return getOrCreatePersistence(client, dbSubDirectory.getDbPath(), fileName, persistableStore); + return getOrCreatePersistence(client, + dbSubDirectory, + fileName, + persistableStore, + MaxBackupSize.from(dbSubDirectory)); + } + + public > Persistence getOrCreatePersistence(PersistenceClient client, + DbSubDirectory dbSubDirectory, + PersistableStore persistableStore, + MaxBackupSize maxBackupSize) { + return getOrCreatePersistence(client, + dbSubDirectory.getDbPath(), + persistableStore.getClass().getSimpleName(), + persistableStore, + maxBackupSize); + } + + public > Persistence getOrCreatePersistence(PersistenceClient client, + DbSubDirectory dbSubDirectory, + String fileName, + PersistableStore persistableStore, + MaxBackupSize maxBackupSize) { + return getOrCreatePersistence(client, + dbSubDirectory.getDbPath(), + fileName, + persistableStore, + maxBackupSize); } public > Persistence getOrCreatePersistence(PersistenceClient client, String subDir, String fileName, - PersistableStore persistableStore) { + PersistableStore persistableStore, + MaxBackupSize maxBackupSize) { PersistableStoreResolver.addResolver(persistableStore.getResolver()); clients.add(client); - Persistence persistence = new Persistence<>(baseDir + File.separator + subDir, fileName); + Persistence persistence = new Persistence<>(baseDir + File.separator + subDir, fileName, maxBackupSize); persistenceInstances.add(persistence); return persistence; } + public CompletableFuture pruneAllBackups() { + List> list = clients.stream() + .map(PersistenceClient::getPersistence) + .map(Persistence::pruneBackups) + .toList(); + return CompletableFutureUtils.allOf(list).thenApply(l -> null); + } + public CompletableFuture readAllPersisted() { List storagePaths = clients.stream() .map(persistenceClient -> persistenceClient.getPersistence().getStorePath() diff --git a/persistence/src/main/java/bisq/persistence/RateLimitedPersistenceClient.java b/persistence/src/main/java/bisq/persistence/RateLimitedPersistenceClient.java index d2e4978836..5d3a8b9305 100644 --- a/persistence/src/main/java/bisq/persistence/RateLimitedPersistenceClient.java +++ b/persistence/src/main/java/bisq/persistence/RateLimitedPersistenceClient.java @@ -58,9 +58,9 @@ public CompletableFuture persist() { dropped = false; return getPersistence() .persistAsync(getPersistableStore().getClone()) - .handle((r, t) -> { + .handle((nil, throwable) -> { writeInProgress = false; - return true; + return throwable == null; }); } } diff --git a/persistence/src/main/java/bisq/persistence/backup/BackupFileInfo.java b/persistence/src/main/java/bisq/persistence/backup/BackupFileInfo.java new file mode 100644 index 0000000000..eabb6fa9f1 --- /dev/null +++ b/persistence/src/main/java/bisq/persistence/backup/BackupFileInfo.java @@ -0,0 +1,87 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.persistence.backup; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.temporal.IsoFields; +import java.util.Optional; + +@Slf4j +@Getter +@ToString +@EqualsAndHashCode +public class BackupFileInfo implements Comparable { + public static Optional from(String fileName, String fileNameWithDate) { + String formattedDate = fileNameWithDate.replace(fileName + "_", ""); + try { + LocalDateTime localDateTime = LocalDateTime.parse(formattedDate, BackupService.DATE_FORMAT); + return Optional.of(new BackupFileInfo(localDateTime, fileNameWithDate)); + } catch (Exception e) { + log.error("Could not resolve date from file {}", fileNameWithDate, e); + return Optional.empty(); + } + } + + private final LocalDateTime localDateTime; + private final String fileNameWithDate; + + public BackupFileInfo(LocalDateTime localDateTime, String fileNameWithDate) { + this.localDateTime = localDateTime; + this.fileNameWithDate = fileNameWithDate; + } + + public LocalDate getLocalDate() { + return localDateTime.toLocalDate(); + } + + public int getMinutes() { + return getLocalDateTime().getMinute(); + } + + public int getHour() { + return getLocalDateTime().getHour(); + } + + public int getDay() { + return getLocalDate().getDayOfWeek().getValue(); + } + + public int getWeek() { + return getLocalDate().get(IsoFields.WEEK_OF_WEEK_BASED_YEAR); + } + + public int getMonth() { + return getLocalDate().getMonthValue(); + } + + public int getYear() { + return getLocalDate().getYear(); + } + + // Sort backups by date, most recent first + @Override + public int compareTo(BackupFileInfo o) { + return o.getLocalDateTime().compareTo(localDateTime); + } +} diff --git a/persistence/src/main/java/bisq/persistence/backup/BackupService.java b/persistence/src/main/java/bisq/persistence/backup/BackupService.java new file mode 100644 index 0000000000..2bc522ed5f --- /dev/null +++ b/persistence/src/main/java/bisq/persistence/backup/BackupService.java @@ -0,0 +1,287 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.persistence.backup; + +import bisq.common.data.ByteUnit; +import bisq.common.file.FileUtils; +import bisq.persistence.Persistence; +import com.google.common.annotations.VisibleForTesting; +import lombok.Setter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Predicate; + +/** + * We back up the persisted data at each write operation. We append the date time format with minutes as smallest time unit. + * Thus, multiple backups during the same minute overwrite the previous one. + * + * Retention based backup strategy: + * - We keep every backup of the last hour with the smallest time unit of 1 minute + * - If older than 1 hour and not older than 24 hours, we keep the newest backup per hour + * - If older than 1 day and not older than 7 days, we keep the newest backup per day + * - If older than 7 days and not older than 28 days, we keep the newest backup per calendar week + * - If older than 28 days but not older than 1 year, we keep the newest backup per month + * - If older than 1 year we keep the newest backup per year + * + * The max number of backups is: 60 + 23 + 6 + 3 + 11 + number of years * 11. for 1 year its: 103. + * Assuming that most data do not get recent updates each minute, we would have about 40-50 backups. + * If the backup file is 600 bytes (Settings), it would result in 61.8 KB. + * If it is 1MB (typical size for user_profile_store.protobuf) it would result in 40-50 MB. + * To avoid too much growth of backups we use the MaxBackupSize and drop old backups once the limit is reached. + * We check as well for the totalMaxBackupSize (sum of all backups of all storage files) and once reached drop backups. + */ +@Slf4j +@ToString +public class BackupService { + static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd_HHmm"); + public static final double TOTAL_MAX_BACKUP_SIZE_IN_MB = 100; + private static final Map accumulatedFileSizeByStore = new ConcurrentHashMap<>(); + @Setter + private static double totalMaxBackupSize = ByteUnit.MB.toBytes(100); + + private final String fileName; + @VisibleForTesting + final Path dirPath; + private final Path storeFilePath; + private final MaxBackupSize maxBackupSize; + + private final Map fileSizeByBackupFileInfo = new HashMap<>(); + private long accumulatedFileSize; + + public BackupService(Path dataDir, Path storeFilePath, MaxBackupSize maxBackupSize) { + this.storeFilePath = storeFilePath; + this.maxBackupSize = maxBackupSize; + + fileName = storeFilePath.getFileName().toString(); + Path backupDir = Path.of(storeFilePath.toString() + .replaceFirst("db", "backups") + .replace(fileName, "")); + String dirName = fileName.replace(Persistence.EXTENSION, "") + .replace("_store", ""); + dirPath = backupDir.resolve(dirName); + } + + public void maybeMigrateLegacyBackupFile() { + if (maxBackupSize == MaxBackupSize.ZERO) { + return; + } + + try { + Path legacyBackupDir = storeFilePath.getParent().resolve("backup"); + File legacyBackupFile = legacyBackupDir.resolve(fileName).toFile(); + if (legacyBackupFile.exists()) { + File newBackupFile = getBackupFile(); + FileUtils.renameFile(legacyBackupFile, newBackupFile); + if (FileUtils.listFiles(legacyBackupDir).isEmpty()) { + FileUtils.deleteFileOrDirectory(legacyBackupDir.toFile()); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public boolean maybeBackup() { + log.error("totalMaxBackupSize {}",totalMaxBackupSize); + if (maxBackupSize == MaxBackupSize.ZERO) { + return false; + } + + if (!storeFilePath.toFile().exists()) { + return false; + } + + // If we get over half of maxBackupSize we prune + long fileSize = updateAndGetAccumulatedFileSize(); + if (fileSize > maxBackupSize.getSizeInBytes() / 2) { + prune(); + } + + try { + return backup(getBackupFile()); + } catch (IOException ex) { + log.error("Backup failed", ex); + return false; + } + } + + @VisibleForTesting + boolean backup(File backupFile) throws IOException { + boolean success = FileUtils.renameFile(storeFilePath.toFile(), backupFile); + if (!success) { + log.error("Could not rename {} to {}", storeFilePath, backupFile); + } + return success; + } + + public void prune() { + if (maxBackupSize == MaxBackupSize.ZERO) { + return; + } + + accumulatedFileSize = 0; + Set fileNames = FileUtils.listFiles(dirPath); + List backupFileInfoList = createBackupFileInfo(fileName, fileNames); + LocalDateTime now = LocalDateTime.now(); + List outdatedBackupFileInfos = findOutdatedBackups(new ArrayList<>(backupFileInfoList), now, this::isMaxFileSizeReached); + outdatedBackupFileInfos.forEach(backupFileInfo -> { + try { + String fileNameWithDate = backupFileInfo.getFileNameWithDate(); + FileUtils.deleteFile(dirPath.resolve(fileNameWithDate).toFile()); + log.info("Deleted outdated backup {}", fileNameWithDate); + } catch (Exception e) { + log.error("Failed to prune backups", e); + } + }); + } + + @VisibleForTesting + static List findOutdatedBackups(List backupFileInfoList, + LocalDateTime now, + Predicate isMaxFileSizeReachedPredicate) { + Map byMinutes = new HashMap<>(); + Map byHour = new HashMap<>(); + Map byDay = new HashMap<>(); + Map byWeek = new HashMap<>(); + Map byMonth = new HashMap<>(); + Map byYear = new HashMap<>(); + + for (BackupFileInfo backupFileInfo : backupFileInfoList) { + long ageInMinutes = getBackupAgeInMinutes(backupFileInfo, now); + long ageInHours = getBackupAgeInHours(backupFileInfo, now); + long ageInDays = getBackupAgeInDays(backupFileInfo, now); + + if (isMaxFileSizeReachedPredicate.test(backupFileInfo)) { + continue; + } + + if (ageInMinutes < 60) { + byMinutes.putIfAbsent(backupFileInfo.getMinutes(), backupFileInfo); + } else if (ageInHours < 24) { + byHour.putIfAbsent(backupFileInfo.getHour(), backupFileInfo); + } else if (ageInDays <= 7) { + byDay.putIfAbsent(backupFileInfo.getDay(), backupFileInfo); + } else if (ageInDays <= 28) { + byWeek.putIfAbsent(backupFileInfo.getWeek(), backupFileInfo); + } else if (ageInDays <= 365) { + byMonth.putIfAbsent(backupFileInfo.getMonth(), backupFileInfo); + } else { + byYear.putIfAbsent(backupFileInfo.getYear(), backupFileInfo); + } + } + + ArrayList remaining = new ArrayList<>() {{ + addAll(byMinutes.values()); + addAll(byHour.values()); + addAll(byDay.values()); + addAll(byWeek.values()); + addAll(byMonth.values()); + addAll(byYear.values()); + }}; + + ArrayList outDated = new ArrayList<>(backupFileInfoList); + outDated.removeAll(remaining); + return outDated; + } + + private long addAndGetAccumulatedFileSize(BackupFileInfo backupFileInfo) { + accumulatedFileSize += getFileSize(backupFileInfo); + accumulatedFileSizeByStore.put(fileName, accumulatedFileSize); + return accumulatedFileSize; + } + + private long updateAndGetAccumulatedFileSize() { + accumulatedFileSize = 0; + Set fileNames = FileUtils.listFiles(dirPath); + createBackupFileInfo(fileName, fileNames) + .forEach(this::addAndGetAccumulatedFileSize); + return accumulatedFileSize; + } + + private boolean isMaxFileSizeReached(BackupFileInfo backupFileInfo) { + accumulatedFileSize = addAndGetAccumulatedFileSize(backupFileInfo); + long totalAccumulatedFileSize = accumulatedFileSizeByStore.values().stream().mapToLong(e -> e).sum(); + return accumulatedFileSize > maxBackupSize.getSizeInBytes() || totalAccumulatedFileSize > totalMaxBackupSize; + } + + private long getFileSize(BackupFileInfo backupFileInfo) { + Path path = dirPath.resolve(backupFileInfo.getFileNameWithDate()); + String key = path.toAbsolutePath().toString(); + fileSizeByBackupFileInfo.computeIfAbsent(key, k -> { + try { + return Files.size(path); + } catch (IOException e) { + log.error("Failed to read file size of {}", path.toAbsolutePath(), e); + return 0L; + } + }); + return fileSizeByBackupFileInfo.get(key); + } + + + //////////////////////////////////////////////////////////////////// + // Utils + //////////////////////////////////////////////////////////////////// + + @VisibleForTesting + File getBackupFile() throws IOException { + return getBackupFile(LocalDateTime.now()); + } + + @VisibleForTesting + File getBackupFile(LocalDateTime localDateTime) throws IOException { + String formattedDate = DATE_FORMAT.format(localDateTime); + String fileNamePath = fileName + "_" + formattedDate; + Path backupFilePath = dirPath.resolve(fileNamePath); + FileUtils.makeDirs(dirPath); + return backupFilePath.toFile(); + } + + @VisibleForTesting + static List createBackupFileInfo(String fileName, Collection fileNames) { + return fileNames.stream() + .filter(fileNameWithDate -> !fileNameWithDate.equals(".DS_Store")) + .map(fileNameWithDate -> BackupFileInfo.from(fileName, fileNameWithDate)) + .filter(Optional::isPresent) + .map(Optional::get) + .sorted() + .toList(); + } + + private static long getBackupAgeInDays(BackupFileInfo backupFileInfo, LocalDateTime now) { + return ChronoUnit.DAYS.between(backupFileInfo.getLocalDate(), now); + } + + private static long getBackupAgeInMinutes(BackupFileInfo backupFileInfo, LocalDateTime now) { + return ChronoUnit.MINUTES.between(backupFileInfo.getLocalDateTime(), now); + } + + private static long getBackupAgeInHours(BackupFileInfo backupFileInfo, LocalDateTime now) { + return ChronoUnit.HOURS.between(backupFileInfo.getLocalDateTime(), now); + } +} diff --git a/persistence/src/main/java/bisq/persistence/backup/MaxBackupSize.java b/persistence/src/main/java/bisq/persistence/backup/MaxBackupSize.java new file mode 100644 index 0000000000..5d531c974f --- /dev/null +++ b/persistence/src/main/java/bisq/persistence/backup/MaxBackupSize.java @@ -0,0 +1,45 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.persistence.backup; + +import bisq.common.data.ByteUnit; +import bisq.persistence.DbSubDirectory; +import lombok.Getter; + +@Getter +public enum MaxBackupSize { + ZERO(0), + TEN_MB(ByteUnit.MB.toBytes(10)), + HUNDRED_MB(ByteUnit.MB.toBytes(100)); + + public static MaxBackupSize from(DbSubDirectory dbSubDirectory) { + return switch (dbSubDirectory) { + case NETWORK_DB -> ZERO; + case CACHE -> ZERO; + case SETTINGS -> TEN_MB; + case PRIVATE -> HUNDRED_MB; + case WALLETS -> HUNDRED_MB; + }; + } + + private final double sizeInBytes; + + MaxBackupSize(double sizeInBytes) { + this.sizeInBytes = sizeInBytes; + } +} diff --git a/persistence/src/test/java/bisq/persistence/PersistableStoreFileManagerTests.java b/persistence/src/test/java/bisq/persistence/PersistableStoreFileManagerTests.java index e4b4310f04..dcb14338f0 100644 --- a/persistence/src/test/java/bisq/persistence/PersistableStoreFileManagerTests.java +++ b/persistence/src/test/java/bisq/persistence/PersistableStoreFileManagerTests.java @@ -49,73 +49,6 @@ void createParentDirIfNotExisting(@TempDir Path tempDir) { assertThat(storePath.getParent()).exists(); } - @Test - void backupCurrentStoreIfBackupNotExisting(@TempDir Path tempDir) throws IOException { - Path storePath = tempDir.resolve("store"); - createEmptyFile(storePath); - - var storeFileManager = new PersistableStoreFileManager(storePath); - storeFileManager.tryToBackupCurrentStoreFile(); - - Path backupFilePath = tempDir.resolve(BACKUP_DIR + "store"); - assertThat(backupFilePath).exists(); - } - - @Test - void backupCurrentStoreIfBackupExists(@TempDir Path tempDir) throws IOException { - Path storePath = tempDir.resolve("store"); - createEmptyFile(storePath); - var storeFileManager = new PersistableStoreFileManager(storePath); - - Path backupFilePath = tempDir.resolve(BACKUP_DIR + "store"); - boolean isSuccess = backupFilePath.toFile().createNewFile(); - assertThat(isSuccess).isTrue(); - - storeFileManager.tryToBackupCurrentStoreFile(); - assertThat(backupFilePath).exists(); - } - - @Test - void backupNotExistingStore(@TempDir Path tempDir) throws IOException { - Path storePath = tempDir.resolve("store"); - var storeFileManager = new PersistableStoreFileManager(storePath); - - Path backupFilePath = tempDir.resolve(BACKUP_DIR + "store"); - createEmptyFile(backupFilePath); - - storeFileManager.tryToBackupCurrentStoreFile(); - assertThat(backupFilePath).exists(); - assertThat(storePath).doesNotExist(); - } - - @Test - void restoreBackupIfCurrentFileNotExisting(@TempDir Path tempDir) throws IOException { - Path storePath = tempDir.resolve("store"); - var storeFileManager = new PersistableStoreFileManager(storePath); - - Path backupFilePath = tempDir.resolve(BACKUP_DIR + "store"); - boolean isSuccess = backupFilePath.toFile().createNewFile(); - assertThat(isSuccess).isTrue(); - - storeFileManager.restoreBackupFileIfCurrentFileNotExisting(); - assertThat(storePath).exists(); - assertThat(backupFilePath).doesNotExist(); - } - - @Test - void restoreBackupIfCurrentFileExists(@TempDir Path tempDir) throws IOException { - Path storePath = tempDir.resolve("store"); - createEmptyFile(storePath); - var storeFileManager = new PersistableStoreFileManager(storePath); - - Path backupFilePath = tempDir.resolve(BACKUP_DIR + "store"); - boolean isSuccess = backupFilePath.toFile().createNewFile(); - assertThat(isSuccess).isTrue(); - - storeFileManager.restoreBackupFileIfCurrentFileNotExisting(); - assertThat(storePath).exists(); - assertThat(backupFilePath).exists(); - } @Test void renameTempFileToCurrentFileIfCurrentNotExisting(@TempDir Path tempDir) throws IOException { diff --git a/persistence/src/test/java/bisq/persistence/backup/BackupServiceTest.java b/persistence/src/test/java/bisq/persistence/backup/BackupServiceTest.java new file mode 100644 index 0000000000..395be88cc9 --- /dev/null +++ b/persistence/src/test/java/bisq/persistence/backup/BackupServiceTest.java @@ -0,0 +1,421 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.persistence.backup; + +import bisq.common.file.FileUtils; +import bisq.common.platform.PlatformUtils; +import bisq.persistence.Persistence; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.*; +import java.util.function.Predicate; + +import static org.junit.jupiter.api.Assertions.*; + +@Slf4j +public class BackupServiceTest { + private BackupService backupService; + private final Path dataDir = PlatformUtils.getUserDataDir().resolve("bisq2_BackupServiceTest"); + private File storeFile; + + @BeforeEach + void setUp() throws IOException { + Path dbDir = dataDir.resolve("db"); + FileUtils.makeDirs(dbDir); + String storeFileName = "test_store" + Persistence.EXTENSION; + Path storeFilePath = dbDir.resolve(storeFileName); + storeFile = storeFilePath.toFile(); + backupService = new BackupService(dataDir, storeFilePath, MaxBackupSize.HUNDRED_MB); + } + + @AfterEach + void tearDown() throws IOException { + FileUtils.deleteFileOrDirectory(dataDir); + } + + @Test + void testBackup() throws IOException { + FileUtils.writeToFile("test", storeFile); + assertTrue(storeFile.exists()); + File backupFile = backupService.getBackupFile(); + assertFalse(backupFile.exists()); + backupService.backup(backupFile); + assertTrue(backupFile.exists()); + assertFalse(storeFile.exists()); + } + + @Test + void testPrune() { + Predicate isMaxFileSizeReachedFunction = e -> false; + List fileNames; + List list; + List outdatedBackupFileInfos; + List remaining; + LocalDateTime now = LocalDateTime.parse("2024-09-15_1755", BackupService.DATE_FORMAT); + String fileName = "test_store.protobuf"; + + + // Empty + fileNames = List.of(); + list = BackupService.createBackupFileInfo(fileName, fileNames); + outdatedBackupFileInfos = BackupService.findOutdatedBackups(new ArrayList<>(list), now, isMaxFileSizeReachedFunction); + assertEquals(0, outdatedBackupFileInfos.size()); + + // Backup date is in the future. We treat backup as daily backup + fileNames = List.of( + "test_store.protobuf_2024-09-15_1755" + ); + now = LocalDateTime.parse("2024-09-14_1755", BackupService.DATE_FORMAT); + list = BackupService.createBackupFileInfo(fileName, fileNames); + outdatedBackupFileInfos = BackupService.findOutdatedBackups(new ArrayList<>(list), now, isMaxFileSizeReachedFunction); + assertEquals(0, outdatedBackupFileInfos.size()); + now = LocalDateTime.parse("2024-09-15_1755", BackupService.DATE_FORMAT); + + // Last 2 minutes. We keep all + fileNames = List.of( + "test_store.protobuf_2024-09-15_1754", + "test_store.protobuf_2024-09-15_1755" + ); + list = BackupService.createBackupFileInfo(fileName, fileNames); + outdatedBackupFileInfos = BackupService.findOutdatedBackups(new ArrayList<>(list), now, isMaxFileSizeReachedFunction); + assertEquals(0, outdatedBackupFileInfos.size()); + + + // After 1 hour we keep only the newest per hour + fileNames = List.of( + "test_store.protobuf_2024-09-15_1653", + "test_store.protobuf_2024-09-15_1654", + "test_store.protobuf_2024-09-15_1655", // remove as + "test_store.protobuf_2024-09-15_1656", + "test_store.protobuf_2024-09-15_1755" + ); + list = BackupService.createBackupFileInfo(fileName, fileNames); + outdatedBackupFileInfos = BackupService.findOutdatedBackups(new ArrayList<>(list), now, isMaxFileSizeReachedFunction); + assertEquals(2, outdatedBackupFileInfos.size()); + assertTrue(outdatedBackupFileInfos.contains(createBackupFileInfo(fileName, fileNames.get(0)))); + assertTrue(outdatedBackupFileInfos.contains(createBackupFileInfo(fileName, fileNames.get(1)))); + + + // At 24 hour we keep only the newest per day + fileNames = List.of( + "test_store.protobuf_2024-09-14_1754", // remove as 14_1755 is newer + "test_store.protobuf_2024-09-14_1755", // use day map + "test_store.protobuf_2024-09-14_1756", // remove as 14_1757 is newer + "test_store.protobuf_2024-09-14_1757", + "test_store.protobuf_2024-09-14_1855", + "test_store.protobuf_2024-09-15_1755" + ); + list = BackupService.createBackupFileInfo(fileName, fileNames); + outdatedBackupFileInfos = BackupService.findOutdatedBackups(new ArrayList<>(list), now, isMaxFileSizeReachedFunction); + assertEquals(2, outdatedBackupFileInfos.size()); + assertTrue(outdatedBackupFileInfos.contains(createBackupFileInfo(fileName, fileNames.get(0)))); + assertTrue(outdatedBackupFileInfos.contains(createBackupFileInfo(fileName, fileNames.get(2)))); + + + // Different day, we keep all as in last week + fileNames = List.of( + "test_store.protobuf_2024-09-14_1755", + "test_store.protobuf_2024-09-15_1755" + ); + list = BackupService.createBackupFileInfo(fileName, fileNames); + outdatedBackupFileInfos = BackupService.findOutdatedBackups(new ArrayList<>(list), now, isMaxFileSizeReachedFunction); + assertEquals(list.get(0).getFileNameWithDate(), fileNames.get(1)); + assertEquals(0, outdatedBackupFileInfos.size()); + + + // Same day, we keep all as in last week + fileNames = List.of( + "test_store.protobuf_2024-09-15_1754", + "test_store.protobuf_2024-09-15_1755", + "test_store.protobuf_2024-09-15_1752" + ); + list = BackupService.createBackupFileInfo(fileName, fileNames); + outdatedBackupFileInfos = BackupService.findOutdatedBackups(new ArrayList<>(list), now, isMaxFileSizeReachedFunction); + assertEquals(0, outdatedBackupFileInfos.size()); + + + // Same day, we keep only newest as it's older than past 7 days + fileNames = List.of( + "test_store.protobuf_2024-09-05_1754", + "test_store.protobuf_2024-09-05_1755", + "test_store.protobuf_2024-09-05_1752" + ); + list = BackupService.createBackupFileInfo(fileName, fileNames); + outdatedBackupFileInfos = BackupService.findOutdatedBackups(new ArrayList<>(list), now, isMaxFileSizeReachedFunction); + assertEquals(2, outdatedBackupFileInfos.size()); + assertTrue(outdatedBackupFileInfos.contains(createBackupFileInfo(fileName, fileNames.get(0)))); + assertTrue(outdatedBackupFileInfos.contains(createBackupFileInfo(fileName, fileNames.get(2)))); + + + // Past 7 days in calendar week + now = LocalDateTime.parse("2024-09-15_1755", BackupService.DATE_FORMAT); + fileNames = List.of( + "test_store.protobuf_2024-09-09_1755", + "test_store.protobuf_2024-09-10_1755", + "test_store.protobuf_2024-09-11_1755", + "test_store.protobuf_2024-09-12_1755", + "test_store.protobuf_2024-09-13_1755", + "test_store.protobuf_2024-09-14_1755", + "test_store.protobuf_2024-09-15_1755" + ); + list = BackupService.createBackupFileInfo(fileName, fileNames); + outdatedBackupFileInfos = BackupService.findOutdatedBackups(new ArrayList<>(list), now, isMaxFileSizeReachedFunction); + assertTrue(outdatedBackupFileInfos.isEmpty()); + + + // Past 7 days crossing the calendar week index + now = LocalDateTime.parse("2024-09-14_1755", BackupService.DATE_FORMAT); + fileNames = List.of( + "test_store.protobuf_2024-09-08_1755", + "test_store.protobuf_2024-09-09_1755", + "test_store.protobuf_2024-09-10_1755", + "test_store.protobuf_2024-09-11_1755", + "test_store.protobuf_2024-09-12_1755", + "test_store.protobuf_2024-09-13_1755", + "test_store.protobuf_2024-09-14_1755" + ); + list = BackupService.createBackupFileInfo(fileName, fileNames); + outdatedBackupFileInfos = BackupService.findOutdatedBackups(new ArrayList<>(list), now, isMaxFileSizeReachedFunction); + assertTrue(outdatedBackupFileInfos.isEmpty()); + + + // Older than 7 days, we keep only newest per week index + now = LocalDateTime.parse("2024-09-23_1755", BackupService.DATE_FORMAT); // monday + fileNames = List.of( + "test_store.protobuf_2024-09-09_1755", // monday + "test_store.protobuf_2024-09-10_1755", + "test_store.protobuf_2024-09-11_1755", + "test_store.protobuf_2024-09-12_1755", + "test_store.protobuf_2024-09-13_1755", + "test_store.protobuf_2024-09-14_1755", + "test_store.protobuf_2024-09-15_1755" // sunday + ); + list = BackupService.createBackupFileInfo(fileName, fileNames); + outdatedBackupFileInfos = BackupService.findOutdatedBackups(new ArrayList<>(list), now, isMaxFileSizeReachedFunction); + assertEquals(6, outdatedBackupFileInfos.size()); + remaining = new ArrayList<>(list); + remaining.removeAll(outdatedBackupFileInfos); + assertTrue(remaining.contains(createBackupFileInfo(fileName, fileNames.get(6)))); + + + // Older than 7 days, we keep only newest per week index + now = LocalDateTime.parse("2024-09-23_1755", BackupService.DATE_FORMAT); // monday + fileNames = List.of( + "test_store.protobuf_2024-09-08_1755", // sunday + "test_store.protobuf_2024-09-09_1755", // monday + "test_store.protobuf_2024-09-10_1755", + "test_store.protobuf_2024-09-11_1755", + "test_store.protobuf_2024-09-12_1755", + "test_store.protobuf_2024-09-13_1755", + "test_store.protobuf_2024-09-14_1755", + "test_store.protobuf_2024-09-15_1755" // sunday + ); + list = BackupService.createBackupFileInfo(fileName, fileNames); + outdatedBackupFileInfos = BackupService.findOutdatedBackups(new ArrayList<>(list), now, isMaxFileSizeReachedFunction); + assertEquals(6, outdatedBackupFileInfos.size()); + remaining = new ArrayList<>(list); + remaining.removeAll(outdatedBackupFileInfos); + assertTrue(remaining.contains(createBackupFileInfo(fileName, fileNames.get(0)))); + assertTrue(remaining.contains(createBackupFileInfo(fileName, fileNames.get(7)))); + assertEquals(2, remaining.size()); + + + // Past 28 days, we keep only newest of each week + now = LocalDateTime.parse("2024-09-30_1755", BackupService.DATE_FORMAT); // monday + fileNames = List.of( + "test_store.protobuf_2024-09-01_1755", + "test_store.protobuf_2024-09-08_1755", + "test_store.protobuf_2024-09-15_1755", // sunday + "test_store.protobuf_2024-09-22_1755" + ); + list = BackupService.createBackupFileInfo(fileName, fileNames); + outdatedBackupFileInfos = BackupService.findOutdatedBackups(new ArrayList<>(list), now, isMaxFileSizeReachedFunction); + assertEquals(0, outdatedBackupFileInfos.size()); + + + // Past 29 days, we keep only newest of each week and older than 28 days for months + now = LocalDateTime.parse("2024-09-30_1755", BackupService.DATE_FORMAT); // monday + fileNames = List.of( + "test_store.protobuf_2024-09-01_1755",// sunday + "test_store.protobuf_2024-09-02_1755", + "test_store.protobuf_2024-09-07_1755", + "test_store.protobuf_2024-09-08_1755",// sunday + "test_store.protobuf_2024-09-09_1755", + "test_store.protobuf_2024-09-14_1755", + "test_store.protobuf_2024-09-15_1755", // sunday + "test_store.protobuf_2024-09-16_1755", + "test_store.protobuf_2024-09-21_1755", + "test_store.protobuf_2024-09-22_1755" // sunday + ); + list = BackupService.createBackupFileInfo(fileName, fileNames); + outdatedBackupFileInfos = BackupService.findOutdatedBackups(new ArrayList<>(list), now, isMaxFileSizeReachedFunction); + assertEquals(6, outdatedBackupFileInfos.size()); + remaining = new ArrayList<>(list); + remaining.removeAll(outdatedBackupFileInfos); + assertTrue(remaining.contains(createBackupFileInfo(fileName, fileNames.get(0)))); + assertTrue(remaining.contains(createBackupFileInfo(fileName, fileNames.get(3)))); + assertTrue(remaining.contains(createBackupFileInfo(fileName, fileNames.get(6)))); + assertTrue(remaining.contains(createBackupFileInfo(fileName, fileNames.get(9)))); + assertEquals(4, remaining.size()); + + + // Past 12 months, we keep the newest one per month + now = LocalDateTime.parse("2024-12-30_1755", BackupService.DATE_FORMAT); // monday + fileNames = List.of( + "test_store.protobuf_2023-12-31_1755", + "test_store.protobuf_2024-01-01_1755", + "test_store.protobuf_2024-01-30_1755", + "test_store.protobuf_2024-02-30_1755", + "test_store.protobuf_2024-03-30_1755", + "test_store.protobuf_2024-04-30_1755", + "test_store.protobuf_2024-05-30_1755", + "test_store.protobuf_2024-06-30_1755", + "test_store.protobuf_2024-07-30_1755", + "test_store.protobuf_2024-08-30_1755", + "test_store.protobuf_2024-09-30_1755", + "test_store.protobuf_2024-10-30_1755", + "test_store.protobuf_2024-11-29_1755", + "test_store.protobuf_2024-11-30_1755" + ); + list = BackupService.createBackupFileInfo(fileName, fileNames); + outdatedBackupFileInfos = BackupService.findOutdatedBackups(new ArrayList<>(list), now, isMaxFileSizeReachedFunction); + assertEquals(2, outdatedBackupFileInfos.size()); + remaining = new ArrayList<>(list); + remaining.removeAll(outdatedBackupFileInfos); + assertTrue(outdatedBackupFileInfos.contains(createBackupFileInfo(fileName, fileNames.get(1)))); + assertTrue(outdatedBackupFileInfos.contains(createBackupFileInfo(fileName, fileNames.get(12)))); + + + // Past years, we keep only the newest per year if older than 1 year + now = LocalDateTime.parse("2024-12-30_1755", BackupService.DATE_FORMAT); // monday + fileNames = List.of( + "test_store.protobuf_2018-12-29_1755", + "test_store.protobuf_2019-12-29_1755", + "test_store.protobuf_2020-11-29_1755", + "test_store.protobuf_2020-12-29_1755", + "test_store.protobuf_2021-12-29_1755", + "test_store.protobuf_2022-12-29_1755", + "test_store.protobuf_2023-12-29_1755" + ); + list = BackupService.createBackupFileInfo(fileName, fileNames); + outdatedBackupFileInfos = BackupService.findOutdatedBackups(new ArrayList<>(list), now, isMaxFileSizeReachedFunction); + assertEquals(1, outdatedBackupFileInfos.size()); + remaining = new ArrayList<>(list); + remaining.removeAll(outdatedBackupFileInfos); + assertTrue(outdatedBackupFileInfos.contains(createBackupFileInfo(fileName, fileNames.get(2)))); + + + // All mixed up + now = LocalDateTime.parse("2024-12-30_1755", BackupService.DATE_FORMAT); // monday + fileNames = List.of( + "test_store.protobuf_2020-10-29_1755", // remove, as newer exist + "test_store.protobuf_2020-11-28_1755", // remove, as newer exist + "test_store.protobuf_2020-11-29_1755", // remove, as newer exist + "test_store.protobuf_2020-12-29_1755", // keep + "test_store.protobuf_2021-12-29_1755", + "test_store.protobuf_2022-12-29_1755", + "test_store.protobuf_2023-12-29_1755", + "test_store.protobuf_2024-01-01_1755",// remove, as newer exist + "test_store.protobuf_2024-01-30_1755",// keep + // past 28 days, keep the newest per week + "test_store.protobuf_2024-12-01_1755", // keep + + "test_store.protobuf_2024-12-02_1755", + "test_store.protobuf_2024-12-03_1755", + "test_store.protobuf_2024-12-08_1755", // keep + + "test_store.protobuf_2024-12-09_1755", + "test_store.protobuf_2024-12-15_1755",// keep + + "test_store.protobuf_2024-12-16_1755", + "test_store.protobuf_2024-12-21_1755", + "test_store.protobuf_2024-12-22_1755",// keep + + // last 7 days, keep one per day + "test_store.protobuf_2024-12-23_1750", // we only use day not exact time to check for past 7 days. so we keep it + "test_store.protobuf_2024-12-24_1755", + "test_store.protobuf_2024-12-25_1755", + "test_store.protobuf_2024-12-26_1755", + "test_store.protobuf_2024-12-28_1755", + "test_store.protobuf_2024-12-29_1750", //remove as 29_1755 is newer + "test_store.protobuf_2024-12-29_1755", + + // Last 24 hours, keep one per hour + "test_store.protobuf_2024-12-29_1756", + "test_store.protobuf_2024-12-30_1555", + "test_store.protobuf_2024-12-30_1654", //remove as 30_1655 is newer + "test_store.protobuf_2024-12-30_1655", + + // Last hour we keep all per minute + "test_store.protobuf_2024-12-30_1656", + "test_store.protobuf_2024-12-30_1754", + "test_store.protobuf_2024-12-30_1755" + ); + list = BackupService.createBackupFileInfo(fileName, fileNames); + outdatedBackupFileInfos = BackupService.findOutdatedBackups(new ArrayList<>(list), now, isMaxFileSizeReachedFunction); + remaining = new ArrayList<>(list); + remaining.removeAll(outdatedBackupFileInfos); + assertEquals(11, outdatedBackupFileInfos.size()); + assertTrue(outdatedBackupFileInfos.contains(createBackupFileInfo(fileName, fileNames.get(0)))); + assertTrue(outdatedBackupFileInfos.contains(createBackupFileInfo(fileName, fileNames.get(1)))); + assertTrue(outdatedBackupFileInfos.contains(createBackupFileInfo(fileName, fileNames.get(2)))); + assertTrue(outdatedBackupFileInfos.contains(createBackupFileInfo(fileName, fileNames.get(7)))); + assertTrue(outdatedBackupFileInfos.contains(createBackupFileInfo(fileName, fileNames.get(10)))); + assertTrue(outdatedBackupFileInfos.contains(createBackupFileInfo(fileName, fileNames.get(11)))); + assertTrue(outdatedBackupFileInfos.contains(createBackupFileInfo(fileName, fileNames.get(13)))); + assertTrue(outdatedBackupFileInfos.contains(createBackupFileInfo(fileName, fileNames.get(15)))); + assertTrue(outdatedBackupFileInfos.contains(createBackupFileInfo(fileName, fileNames.get(16)))); + assertTrue(outdatedBackupFileInfos.contains(createBackupFileInfo(fileName, fileNames.get(23)))); + assertTrue(outdatedBackupFileInfos.contains(createBackupFileInfo(fileName, fileNames.get(27)))); + } + + void createBackups() throws IOException { + Calendar calendar = GregorianCalendar.getInstance(); + calendar.setTime(new Date()); + int writeFrequency = 60 * 60 * 24; // every hour + int seconds = 60; + int minutes = 60; + int hours = 24; + int days = 1000; + long ts = System.currentTimeMillis(); + int numItems = seconds * minutes * hours * days / writeFrequency; + for (int i = 0; i < numItems; i++) { + calendar.add(Calendar.SECOND, -writeFrequency); + Date calendarTime = calendar.getTime(); + LocalDateTime localDateTime = LocalDateTime.ofInstant(calendarTime.toInstant(), ZoneId.systemDefault()); + File backupFile = backupService.getBackupFile(localDateTime); + // As we rename the storage file at backup we need to create it before backup. + FileUtils.writeToFile("test", storeFile); + backupService.backup(backupFile); + } + log.error("createBackups took {} ms", System.currentTimeMillis() - ts); // seconds * minutes * hours -> took 16275 ms + } + + + private static BackupFileInfo createBackupFileInfo(String fileName, String fileNameWithDate) { + return BackupFileInfo.from(fileName, fileNameWithDate).orElseThrow(); + } +} diff --git a/settings/src/main/java/bisq/settings/SettingsService.java b/settings/src/main/java/bisq/settings/SettingsService.java index 977ff2bf09..cdf5792619 100644 --- a/settings/src/main/java/bisq/settings/SettingsService.java +++ b/settings/src/main/java/bisq/settings/SettingsService.java @@ -20,6 +20,7 @@ import bisq.common.application.Service; import bisq.common.currency.FiatCurrencyRepository; import bisq.common.currency.Market; +import bisq.common.data.ByteUnit; import bisq.common.locale.CountryRepository; import bisq.common.locale.LanguageRepository; import bisq.common.locale.LocaleRepository; @@ -30,6 +31,7 @@ import bisq.persistence.Persistence; import bisq.persistence.PersistenceClient; import bisq.persistence.PersistenceService; +import bisq.persistence.backup.BackupService; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -56,7 +58,9 @@ public class SettingsService implements PersistenceClient, Servic private boolean isInitialized; public SettingsService(PersistenceService persistenceService) { - persistence = persistenceService.getOrCreatePersistence(this, DbSubDirectory.SETTINGS, persistableStore); + persistence = persistenceService.getOrCreatePersistence(this, + DbSubDirectory.SETTINGS, + persistableStore); SettingsService.instance = this; } @@ -88,6 +92,10 @@ public CompletableFuture initialize() { getShowMarketSelectionListCollapsed().addObserver(value -> persist()); getBackupLocation().addObserver(value -> persist()); getShowMyOffersOnly().addObserver(value -> persist()); + getTotalMaxBackupSizeInMB().addObserver(value -> { + BackupService.setTotalMaxBackupSize(ByteUnit.MB.toBytes(value)); + persist(); + }); isInitialized = true; @@ -102,8 +110,7 @@ public CompletableFuture shutdown() { public CompletableFuture persist() { // We don't want to call persist from the addObserver calls at initialize if (isInitialized) { - return getPersistence().persistAsync(getPersistableStore().getClone()) - .handle((r, t) -> true); + return PersistenceClient.super.persist(); } else { return CompletableFuture.completedFuture(true); } @@ -212,6 +219,10 @@ public Observable getShowMyOffersOnly() { return persistableStore.showMyOffersOnly; } + public Observable getTotalMaxBackupSizeInMB() { + return persistableStore.totalMaxBackupSizeInMB; + } + /////////////////////////////////////////////////////////////////////////////////////////////////// // DontShowAgainMap diff --git a/settings/src/main/java/bisq/settings/SettingsStore.java b/settings/src/main/java/bisq/settings/SettingsStore.java index c9c19a5a52..556ca632ee 100644 --- a/settings/src/main/java/bisq/settings/SettingsStore.java +++ b/settings/src/main/java/bisq/settings/SettingsStore.java @@ -27,6 +27,7 @@ import bisq.common.proto.UnresolvableProtobufMessageException; import bisq.network.p2p.node.network_load.NetworkLoad; import bisq.persistence.PersistableStore; +import bisq.persistence.backup.BackupService; import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; @@ -62,6 +63,7 @@ public final class SettingsStore implements PersistableStore { final Observable showMarketSelectionListCollapsed = new Observable<>(); final Observable backupLocation = new Observable<>(); final Observable showMyOffersOnly = new Observable<>(); + final Observable totalMaxBackupSizeInMB = new Observable<>(); public SettingsStore() { this(new Cookie(), @@ -87,7 +89,8 @@ public SettingsStore() { false, false, PlatformUtils.getHomeDirectory(), - false); + false, + BackupService.TOTAL_MAX_BACKUP_SIZE_IN_MB); } public SettingsStore(Cookie cookie, @@ -113,7 +116,8 @@ public SettingsStore(Cookie cookie, boolean showOfferListExpanded, boolean showMarketSelectionListCollapsed, String backupLocation, - boolean showMyOffersOnly) { + boolean showMyOffersOnly, + double totalMaxBackupSizeInMB) { this.cookie = cookie; this.dontShowAgainMap.putAll(dontShowAgainMap); this.useAnimations.set(useAnimations); @@ -138,6 +142,7 @@ public SettingsStore(Cookie cookie, this.showMarketSelectionListCollapsed.set(showMarketSelectionListCollapsed); this.backupLocation.set(backupLocation); this.showMyOffersOnly.set(showMyOffersOnly); + this.totalMaxBackupSizeInMB.set(totalMaxBackupSizeInMB); } @Override @@ -166,7 +171,8 @@ public bisq.settings.protobuf.SettingsStore.Builder getBuilder(boolean serialize .setShowOfferListExpanded(showOfferListExpanded.get()) .setShowMarketSelectionListCollapsed(showMarketSelectionListCollapsed.get()) .setBackupLocation(backupLocation.get()) - .setShowMyOffersOnly(showMyOffersOnly.get()); + .setShowMyOffersOnly(showMyOffersOnly.get()) + .setTotalMaxBackupSizeInMB(totalMaxBackupSizeInMB.get()); } @Override @@ -182,6 +188,10 @@ public static SettingsStore fromProto(bisq.settings.protobuf.SettingsStore proto if (maxTradePriceDeviation == 0) { maxTradePriceDeviation = SettingsService.DEFAULT_MAX_TRADE_PRICE_DEVIATION; } + double totalMaxBackupSizeInMB = proto.getTotalMaxBackupSizeInMB(); + if (totalMaxBackupSizeInMB == 0) { + totalMaxBackupSizeInMB = BackupService.TOTAL_MAX_BACKUP_SIZE_IN_MB; + } return new SettingsStore(Cookie.fromProto(proto.getCookie()), proto.getDontShowAgainMapMap().entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)), @@ -207,7 +217,8 @@ public static SettingsStore fromProto(bisq.settings.protobuf.SettingsStore proto proto.getShowOfferListExpanded(), proto.getShowMarketSelectionListCollapsed(), proto.getBackupLocation(), - proto.getShowMyOffersOnly()); + proto.getShowMyOffersOnly(), + totalMaxBackupSizeInMB); } @Override @@ -246,7 +257,8 @@ public SettingsStore getClone() { showOfferListExpanded.get(), showMarketSelectionListCollapsed.get(), backupLocation.get(), - showMyOffersOnly.get()); + showMyOffersOnly.get(), + totalMaxBackupSizeInMB.get()); } @Override @@ -276,6 +288,7 @@ public void applyPersisted(SettingsStore persisted) { showMarketSelectionListCollapsed.set(persisted.showMarketSelectionListCollapsed.get()); backupLocation.set(persisted.backupLocation.get()); showMyOffersOnly.set(persisted.showMyOffersOnly.get()); + totalMaxBackupSizeInMB.set(persisted.totalMaxBackupSizeInMB.get()); } catch (Exception e) { log.error("Exception at applyPersisted", e); } diff --git a/settings/src/main/proto/settings.proto b/settings/src/main/proto/settings.proto index 3d72cdb028..b882df460e 100644 --- a/settings/src/main/proto/settings.proto +++ b/settings/src/main/proto/settings.proto @@ -63,4 +63,5 @@ message SettingsStore { bool showMarketSelectionListCollapsed = 22; string backupLocation = 23; bool showMyOffersOnly = 24; + double totalMaxBackupSizeInMB = 25; }