Skip to content

Commit

Permalink
fetch only ids to improve performance during cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
Athou committed Jul 1, 2023
1 parent 82cf0e1 commit 922346b
Show file tree
Hide file tree
Showing 5 changed files with 24 additions and 22 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,10 @@ public List<FeedEntryContent> findExisting(String contentHash, String titleHash)
return query().select(content).from(content).where(content.contentHash.eq(contentHash), content.titleHash.eq(titleHash)).fetch();
}

public int deleteWithoutEntries(int max) {

public long deleteWithoutEntries(int max) {
JPQLQuery<Integer> subQuery = JPAExpressions.selectOne().from(entry).where(entry.content.id.eq(content.id));
List<FeedEntryContent> list = query().selectFrom(content).where(subQuery.notExists()).limit(max).fetch();
List<Long> ids = query().select(content.id).from(content).where(subQuery.notExists()).limit(max).fetch();

int deleted = list.size();
delete(list);
return deleted;
return deleteQuery(content).where(content.id.in(ids)).execute();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ public List<FeedCapacity> findFeedsExceedingCapacity(long maxCapacity, long max)
}

public int delete(Long feedId, long max) {

List<FeedEntry> list = query().selectFrom(entry).where(entry.feed.id.eq(feedId)).limit(max).fetch();
return delete(list);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -270,8 +270,13 @@ private List<FeedEntryStatus> lazyLoadContent(boolean includeContent, List<FeedE
return results;
}

public List<FeedEntryStatus> getOldStatuses(Date olderThan, int limit) {
return query().selectFrom(status).where(status.entryInserted.lt(olderThan), status.starred.isFalse()).limit(limit).fetch();
public long deleteOldStatuses(Date olderThan, int limit) {
List<Long> ids = query().select(status.id)
.from(status)
.where(status.entryInserted.lt(olderThan), status.starred.isFalse())
.limit(limit)
.fetch();
return deleteQuery(status).where(status.id.in(ids)).execute();
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import com.commafeed.backend.model.AbstractModel;
import com.querydsl.core.types.EntityPath;
import com.querydsl.jpa.impl.JPADeleteClause;
import com.querydsl.jpa.impl.JPAQuery;
import com.querydsl.jpa.impl.JPAQueryFactory;
import com.querydsl.jpa.impl.JPAUpdateClause;
Expand All @@ -30,6 +31,10 @@ protected JPAUpdateClause updateQuery(EntityPath<T> entityPath) {
return new JPAUpdateClause(currentSession(), entityPath);
}

protected JPADeleteClause deleteQuery(EntityPath<T> entityPath) {
return new JPADeleteClause(currentSession(), entityPath);
}

public void saveOrUpdate(T model) {
persist(model);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,15 +34,15 @@ public class DatabaseCleaningService {
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;

public long cleanFeedsWithoutSubscriptions() {
public void cleanFeedsWithoutSubscriptions() {
log.info("cleaning feeds without subscriptions");
long total = 0;
int deleted = 0;
int deleted;
long entriesTotal = 0;
do {
List<Feed> feeds = unitOfWork.call(() -> feedDAO.findWithoutSubscriptions(1));
for (Feed feed : feeds) {
int entriesDeleted = 0;
long entriesDeleted;
do {
entriesDeleted = unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), BATCH_SIZE));
entriesTotal += entriesDeleted;
Expand All @@ -54,23 +54,21 @@ public long cleanFeedsWithoutSubscriptions() {
log.info("removed {} feeds without subscriptions", total);
} while (deleted != 0);
log.info("cleanup done: {} feeds without subscriptions deleted", total);
return total;
}

public long cleanContentsWithoutEntries() {
public void cleanContentsWithoutEntries() {
log.info("cleaning contents without entries");
long total = 0;
int deleted = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(BATCH_SIZE));
total += deleted;
log.info("removed {} contents without entries", total);
} while (deleted != 0);
log.info("cleanup done: {} contents without entries deleted", total);
return total;
}

public long cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
long total = 0;
while (true) {
List<FeedCapacity> feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, BATCH_SIZE));
Expand All @@ -90,19 +88,17 @@ public long cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
}
}
log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total);
return total;
}

public long cleanStatusesOlderThan(final Date olderThan) {
public void cleanStatusesOlderThan(final Date olderThan) {
log.info("cleaning old read statuses");
long total = 0;
int deleted = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryStatusDAO.delete(feedEntryStatusDAO.getOldStatuses(olderThan, BATCH_SIZE)));
deleted = unitOfWork.call(() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, BATCH_SIZE));
total += deleted;
log.info("removed {} old read statuses", total);
} while (deleted != 0);
log.info("cleanup done: {} old read statuses deleted", total);
return total;
}
}

0 comments on commit 922346b

Please sign in to comment.