From 922346bef6a7b447d438141abb960f22746b2904 Mon Sep 17 00:00:00 2001 From: Athou Date: Fri, 30 Jun 2023 08:54:05 +0200 Subject: [PATCH] fetch only ids to improve performance during cleanup --- .../backend/dao/FeedEntryContentDAO.java | 9 +++----- .../commafeed/backend/dao/FeedEntryDAO.java | 1 - .../backend/dao/FeedEntryStatusDAO.java | 9 ++++++-- .../com/commafeed/backend/dao/GenericDAO.java | 5 +++++ .../service/DatabaseCleaningService.java | 22 ++++++++----------- 5 files changed, 24 insertions(+), 22 deletions(-) diff --git a/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryContentDAO.java b/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryContentDAO.java index 0e2ec6606..535eb0d3c 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryContentDAO.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryContentDAO.java @@ -28,13 +28,10 @@ public List findExisting(String contentHash, String titleHash) return query().select(content).from(content).where(content.contentHash.eq(contentHash), content.titleHash.eq(titleHash)).fetch(); } - public int deleteWithoutEntries(int max) { - + public long deleteWithoutEntries(int max) { JPQLQuery subQuery = JPAExpressions.selectOne().from(entry).where(entry.content.id.eq(content.id)); - List list = query().selectFrom(content).where(subQuery.notExists()).limit(max).fetch(); + List ids = query().select(content.id).from(content).where(subQuery.notExists()).limit(max).fetch(); - int deleted = list.size(); - delete(list); - return deleted; + return deleteQuery(content).where(content.id.in(ids)).execute(); } } diff --git a/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java b/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java index 69f0d37d4..3c7af8c0b 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java @@ -48,7 +48,6 @@ public List findFeedsExceedingCapacity(long maxCapacity, long max) } public int delete(Long feedId, long max) { - List list = query().selectFrom(entry).where(entry.feed.id.eq(feedId)).limit(max).fetch(); return delete(list); } diff --git a/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java b/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java index 4eebc2569..cb2e4d4cb 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java @@ -270,8 +270,13 @@ private List lazyLoadContent(boolean includeContent, List getOldStatuses(Date olderThan, int limit) { - return query().selectFrom(status).where(status.entryInserted.lt(olderThan), status.starred.isFalse()).limit(limit).fetch(); + public long deleteOldStatuses(Date olderThan, int limit) { + List ids = query().select(status.id) + .from(status) + .where(status.entryInserted.lt(olderThan), status.starred.isFalse()) + .limit(limit) + .fetch(); + return deleteQuery(status).where(status.id.in(ids)).execute(); } } diff --git a/commafeed-server/src/main/java/com/commafeed/backend/dao/GenericDAO.java b/commafeed-server/src/main/java/com/commafeed/backend/dao/GenericDAO.java index ffd1b7dea..d19315462 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/dao/GenericDAO.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/dao/GenericDAO.java @@ -7,6 +7,7 @@ import com.commafeed.backend.model.AbstractModel; import com.querydsl.core.types.EntityPath; +import com.querydsl.jpa.impl.JPADeleteClause; import com.querydsl.jpa.impl.JPAQuery; import com.querydsl.jpa.impl.JPAQueryFactory; import com.querydsl.jpa.impl.JPAUpdateClause; @@ -30,6 +31,10 @@ protected JPAUpdateClause updateQuery(EntityPath entityPath) { return new JPAUpdateClause(currentSession(), entityPath); } + protected JPADeleteClause deleteQuery(EntityPath entityPath) { + return new JPADeleteClause(currentSession(), entityPath); + } + public void saveOrUpdate(T model) { persist(model); } diff --git a/commafeed-server/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java b/commafeed-server/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java index 4f3f8913d..2155d60e2 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java @@ -34,15 +34,15 @@ public class DatabaseCleaningService { private final FeedEntryContentDAO feedEntryContentDAO; private final FeedEntryStatusDAO feedEntryStatusDAO; - public long cleanFeedsWithoutSubscriptions() { + public void cleanFeedsWithoutSubscriptions() { log.info("cleaning feeds without subscriptions"); long total = 0; - int deleted = 0; + int deleted; long entriesTotal = 0; do { List feeds = unitOfWork.call(() -> feedDAO.findWithoutSubscriptions(1)); for (Feed feed : feeds) { - int entriesDeleted = 0; + long entriesDeleted; do { entriesDeleted = unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), BATCH_SIZE)); entriesTotal += entriesDeleted; @@ -54,23 +54,21 @@ public long cleanFeedsWithoutSubscriptions() { log.info("removed {} feeds without subscriptions", total); } while (deleted != 0); log.info("cleanup done: {} feeds without subscriptions deleted", total); - return total; } - public long cleanContentsWithoutEntries() { + public void cleanContentsWithoutEntries() { log.info("cleaning contents without entries"); long total = 0; - int deleted = 0; + long deleted; do { deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(BATCH_SIZE)); total += deleted; log.info("removed {} contents without entries", total); } while (deleted != 0); log.info("cleanup done: {} contents without entries deleted", total); - return total; } - public long cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) { + public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) { long total = 0; while (true) { List feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, BATCH_SIZE)); @@ -90,19 +88,17 @@ public long cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) { } } log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total); - return total; } - public long cleanStatusesOlderThan(final Date olderThan) { + public void cleanStatusesOlderThan(final Date olderThan) { log.info("cleaning old read statuses"); long total = 0; - int deleted = 0; + long deleted; do { - deleted = unitOfWork.call(() -> feedEntryStatusDAO.delete(feedEntryStatusDAO.getOldStatuses(olderThan, BATCH_SIZE))); + deleted = unitOfWork.call(() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, BATCH_SIZE)); total += deleted; log.info("removed {} old read statuses", total); } while (deleted != 0); log.info("cleanup done: {} old read statuses deleted", total); - return total; } }