refactoring purge task and cli

lib/bin/purge-forms.js → lib/bin/purge.js

@@ -7,20 +7,32 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 //
-// This script checks for (soft-)deleted forms and purges any that were deleted
-// over 30 days ago.
+// This script checks for (soft-)deleted forms and submissions and purges
+// any that were deleted over 30 days ago.
+//
+// It also accepts command line arguments that can force the purging of
+// forms and submissions that were deleted less than 30 days ago.
+//
+// It can also be used to purge a specific form or submission
+// (that has already been marked deleted).
 
 const { run } = require('../task/task');
-const { purgeForms } = require('../task/purge');
+const { purgeForms, purgeSubmissions, purgeUnattachedBlobs } = require('../task/purge');
 
 const { program } = require('commander');
 program.option('-f, --force', 'Force any soft-deleted form to be purged right away.');
 program.option('-i, --formId <integer>', 'Purge a specific form based on its id.', parseInt);
 program.option('-p, --projectId <integer>', 'Restrict purging to a specific project.', parseInt);
-program.option('-x, --xmlFormId <value>', 'Restrict purging to specific form based on xmlFormId (must be used with project id).');
+program.option('-x, --xmlFormId <value>', 'Restrict purging to specific form based on xmlFormId (must be used with projectId).');
+program.option('-s, --instanceId <value>', 'Restrict purging to a specific submission based on instanceId (use with projectId and xmlFormId).');
+
 program.parse();
 
 const options = program.opts();
 
-run(purgeForms(options.force, options.formId, options.projectId, options.xmlFormId)
-  .then((count) => `Forms purged: ${count}`));
+const purgeTasks = options.instanceId
+  ? purgeSubmissions(options.force, options.projectId, options.xmlFormId, options.submissionId)
+  : purgeForms(options.force, options.formId, options.projectId, options.xmlFormId);
+
+run(purgeTasks
+  .then((message) => purgeUnattachedBlobs().then(() => message)));

lib/model/query/forms.js

@@ -406,7 +406,7 @@ const _trashedFilter = (force, id, projectId, xmlFormId) => {
 // 3. Update actees table for the specific form to leave some useful information behind
 // 4. Delete the forms and their resources from the database
 // 5. Purge unattached blobs
-const purge = (force = false, id = null, projectId = null, xmlFormId = null) => ({ oneFirst, Blobs }) => {
+const purge = (force = false, id = null, projectId = null, xmlFormId = null) => ({ oneFirst }) => {
   if (xmlFormId != null && projectId == null)
     throw Problem.internal.unknown({ error: 'Must also specify projectId when using xmlFormId' });
   return oneFirst(sql`
@@ -446,9 +446,7 @@ with redacted_audits as (
     where ${_trashedFilter(force, id, projectId, xmlFormId)}
     returning 1
   )
-select count(*) from deleted_forms`)
-    .then((count) => Blobs.purgeUnattached()
-      .then(() => Promise.resolve(count)));
+select count(*) from deleted_forms`);
 };
 
 ////////////////////////////////////////////////////////////////////////////////

lib/model/query/submissions.js

@@ -400,42 +400,43 @@ const DAY_RANGE = config.has('default.taskSchedule.purge')
   ? config.get('default.taskSchedule.purge')
   : 30; // Default is 30 days
 
-const _trashedFilter = (force, id) => {
-  const idFilter = (id
-    ? sql`and submissions.id = ${id}`
+const _trashedFilter = (force, projectId, xmlFormId, instanceId) => {
+  const idFilter = ((instanceId != null) && (projectId != null) && (xmlFormId != null)
+    ? sql`and submissions."instanceId" = ${instanceId}
+    and forms."projectId" = ${projectId}
+    and forms."xmlFormId" = ${xmlFormId}`
     : sql``);
   return (force
     ? sql`submissions."deletedAt" is not null ${idFilter}`
     : sql`submissions."deletedAt" < current_date - cast(${DAY_RANGE} as int) ${idFilter}`);
 };
 
-// eslint-disable-next-line no-unused-vars
-const purge = (force = false, id = null) => ({ oneFirst, Blobs }) =>
+const purge = (force = false, projectId = null, xmlFormId = null, instanceId = null) => ({ oneFirst }) =>
   oneFirst(sql`
 with redacted_audits as (
     update audits set notes = ''
     from submissions
     where (audits.details->>'submissionId')::int = submissions.id
-    and ${_trashedFilter(force, id)}
+    and ${_trashedFilter(force, projectId, xmlFormId, instanceId)}
   ), deleted_client_audits as (
     delete from client_audits
     using submission_attachments, submission_defs, submissions
     where client_audits."blobId" = submission_attachments."blobId"
     and submission_attachments."submissionDefId" = submission_defs.id
     and submission_attachments."isClientAudit" = true
     and submission_defs."submissionId" = submissions.id
-    and ${_trashedFilter(force, id)}
+    and ${_trashedFilter(force, projectId, xmlFormId, instanceId)}
   ), purge_audits as (
     insert into audits ("action", "loggedAt", "processed")
     values ('submission.purge', clock_timestamp(), clock_timestamp())
   ), deleted_submissions as (
     delete from submissions
-    where ${_trashedFilter(force, id)}
-    returning *
+    using forms
+    where submissions."formId" = forms.id
+    and ${_trashedFilter(force, projectId, xmlFormId, instanceId)}
+    returning submissions.*
   )
-select count(*) from deleted_submissions`)
-    .then((count) => Blobs.purgeUnattached()
-      .then(() => Promise.resolve(count)));
+select count(*) from deleted_submissions`);
 
 module.exports = {
   createNew, createVersion,

lib/task/purge.js

@@ -10,9 +10,14 @@
 const { task } = require('./task');
 
 const purgeForms = task.withContainer(({ Forms }) => (force = false, formId = null, projectId = null, xmlFormId = null) =>
-  Forms.purge(force, formId, projectId, xmlFormId));
+  Forms.purge(force, formId, projectId, xmlFormId)
+    .then((count) => `Forms purged: ${count}`));
 
-const purgeSubmissions = task.withContainer(({ Submissions }) => (force = false, submissionId = null) =>
-  Submissions.purge(force, submissionId));
+const purgeSubmissions = task.withContainer(({ Submissions }) => async (force = false, projectId = null, xmlFormId = null, submissionId = null) => {
+  const count = await Submissions.purge(force, projectId, xmlFormId, submissionId);
+  return `Submissions purged: ${count}`;
+});
 
-module.exports = { purgeForms, purgeSubmissions };
+const purgeUnattachedBlobs = task.withContainer(({ Blobs }) => () => Blobs.purgeUnattached());
+
+module.exports = { purgeForms, purgeSubmissions, purgeUnattachedBlobs };

test/integration/other/blobs.js

@@ -43,6 +43,7 @@ describe('blob query module', () => {
           .expect(201))
         .then(() => asAlice.delete('/v1/projects/1/forms/binaryType'))
         .then(() => container.Forms.purge(true))
+        .then(() => container.Blobs.purgeUnattached())
         .then(() => container.oneFirst(sql`select count(*) from blobs`))
         .then((count) => count.should.equal(1))))); //
 });

test/integration/other/form-purging.js

@@ -1,6 +1,7 @@
 const { createReadStream, readFileSync } = require('fs');
 const appPath = require('app-root-path');
 const { sql } = require('slonik');
+const assert = require('assert');
 const { testService } = require('../setup');
 const testData = require('../../data/xml');
 const { exhaust } = require(appPath + '/lib/worker/worker');
@@ -59,25 +60,6 @@ describe('query module form purge', () => {
             counts.should.eql([ 0, 0 ]);
           })))));
 
-  it('should purge a deleted form by ID', testService((service, container) =>
-    service.login('alice', (asAlice) =>
-      asAlice.delete('/v1/projects/1/forms/simple')
-        .expect(200)
-        .then(() => asAlice.post('/v1/projects/1/forms')
-          .send(testData.forms.withAttachments)
-          .set('Content-Type', 'application/xml')
-          .expect(200))
-        .then(() => container.Forms.getByProjectAndXmlFormId(1, 'withAttachments').then((o) => o.get()))
-        .then((ghostForm) => asAlice.delete('/v1/projects/1/withAttachments')
-          .then(() => container.Forms.purge(true, 1)) // force delete a single form
-          .then(() => Promise.all([
-            container.oneFirst(sql`select count(*) from forms where id = ${ghostForm.id}`),
-            container.oneFirst(sql`select count(*) from forms where id = 1`), // deleted form id
-          ])
-            .then((counts) => {
-              counts.should.eql([ 1, 0 ]);
-            }))))));
-
   it('should log the purge action in the audit log', testService((service, container) =>
     service.login('alice', (asAlice) =>
       container.Forms.getByProjectAndXmlFormId(1, 'simple').then((o) => o.get()) // get the form before we delete it
@@ -142,6 +124,7 @@ describe('query module form purge', () => {
         .then((ghostForm) => asAlice.delete('/v1/projects/1/forms/withAttachments')
           .expect(200)
           .then(() => container.Forms.purge(true))
+          .then(() => container.Blobs.purgeUnattached())
           .then(() => Promise.all([
             container.oneFirst(sql`select count(*) from forms where id = ${ghostForm.id}`),
             container.oneFirst(sql`select count(*) from form_defs where "formId" = ${ghostForm.id}`),
@@ -184,6 +167,97 @@ describe('query module form purge', () => {
         .then(() => container.oneFirst(sql`select count(*) from form_field_values`))
         .then((count) => count.should.eql(0)))));
 
+  describe('purging specific forms via specific arguments', () => {
+    it('should purge a deleted form by ID', testService((service, container) =>
+      service.login('alice', (asAlice) =>
+        asAlice.delete('/v1/projects/1/forms/simple')
+          .expect(200)
+          .then(() => asAlice.post('/v1/projects/1/forms')
+            .send(testData.forms.withAttachments)
+            .set('Content-Type', 'application/xml')
+            .expect(200))
+          .then(() => container.Forms.getByProjectAndXmlFormId(1, 'withAttachments').then((o) => o.get()))
+          .then((ghostForm) => asAlice.delete('/v1/projects/1/withAttachments')
+            .then(() => container.Forms.purge(true, 1)) // force delete a single form
+            .then(() => Promise.all([
+              container.oneFirst(sql`select count(*) from forms where id = ${ghostForm.id}`),
+              container.oneFirst(sql`select count(*) from forms where id = 1`), // deleted form id
+            ])
+              .then((counts) => {
+                counts.should.eql([ 1, 0 ]);
+              }))))));
+
+    it('should purge all versions of deleted form in project', testService(async (service, container) => {
+      const asAlice = await service.login('alice');
+
+      await asAlice.delete('/v1/projects/1/forms/simple')
+        .expect(200);
+
+      // new version (will be v2)
+      await asAlice.post('/v1/projects/1/forms?ignoreWarnings=true')
+        .send(testData.forms.simple)
+        .set('Content-Type', 'application/xml')
+        .expect(200);
+
+      // publish new version v2
+      await asAlice.post('/v1/projects/1/forms/simple/draft/publish?ignoreWarnings=true&version=v2')
+        .expect(200);
+
+      // delete new version v2
+      await asAlice.delete('/v1/projects/1/forms/simple')
+        .expect(200);
+
+      // new version (will be v3)
+      await asAlice.post('/v1/projects/1/forms?ignoreWarnings=true')
+        .send(testData.forms.simple)
+        .set('Content-Type', 'application/xml')
+        .expect(200);
+
+      // publish new version v3 but don't delete
+      await asAlice.post('/v1/projects/1/forms/simple/draft/publish?ignoreWarnings=true&version=v3')
+        .expect(200);
+
+      const count = await container.Forms.purge(true, null, 1, 'simple');
+      count.should.equal(2);
+    }));
+
+    it('should purge named form only from specified project', testService(async (service, container) => {
+      const asAlice = await service.login('alice');
+
+      // delete simple form in project 1 (but don't purge it)
+      await asAlice.delete('/v1/projects/1/forms/simple')
+        .expect(200);
+
+      const newProjectId = await asAlice.post('/v1/projects')
+        .send({ name: 'Project Two' })
+        .then(({ body }) => body.id);
+
+      await asAlice.post(`/v1/projects/${newProjectId}/forms?publish=true`)
+        .send(testData.forms.simple)
+        .set('Content-Type', 'application/xml')
+        .expect(200);
+
+      await asAlice.delete(`/v1/projects/${newProjectId}/forms/simple`)
+        .expect(200);
+
+      const count = await container.Forms.purge(true, null, newProjectId, 'simple');
+      count.should.equal(1);
+    }));
+
+    it('should throw an error when xmlFormId specified without project ID', testService(async (service, container) => {
+      const asAlice = await service.login('alice');
+
+      await asAlice.delete('/v1/projects/1/forms/simple')
+        .expect(200);
+
+      await assert.throws(() => { container.Forms.purge(true, null, null, 'simple'); }, (err) => {
+        err.problemCode.should.equal(500.1);
+        err.problemDetails.error.should.equal('Must also specify projectId when using xmlFormId');
+        return true;
+      });
+    }));
+  });
+
   describe('purging form submissions', () => {
     const withSimpleIds = (deprecatedId, instanceId) => testData.instances.simple.one
       .replace('one</instance', `${instanceId}</instanceID><deprecatedID>${deprecatedId}</deprecated`);
@@ -227,6 +301,7 @@ describe('query module form purge', () => {
             }))
           .then(() => asAlice.delete('/v1/projects/1/forms/binaryType'))
           .then(() => container.Forms.purge(true))
+          .then(() => container.Blobs.purgeUnattached())
           .then(() => container.oneFirst(sql`select count(*) from submission_attachments`)
             .then((count) => count.should.equal(0)))
           .then(() => container.oneFirst(sql`select count(*) from blobs`)
@@ -280,6 +355,7 @@ describe('query module form purge', () => {
           .then(() => exhaust(container))
           .then(() => asAlice.delete('/v1/projects/1/forms/audits'))
           .then(() => container.Forms.purge(true))
+          .then(() => container.Blobs.purgeUnattached())
           .then(() => Promise.all([
             container.oneFirst(sql`select count(*) from client_audits`),
             container.oneFirst(sql`select count(*) from blobs`)
@@ -294,6 +370,7 @@ describe('query module form purge', () => {
           .then(() => asAlice.delete('/v1/projects/1/forms/simple2') // Delete form
             .expect(200))
           .then(() => container.Forms.purge(true))
+          .then(() => container.Blobs.purgeUnattached())
           .then(() => container.oneFirst(sql`select count(*) from blobs`))
           .then((count) => count.should.equal(0)))));
   });

test/integration/other/submission-purging.js

@@ -104,7 +104,7 @@ describe('query module submission purge', () => {
     attachments.should.equal(0);
   }));
 
-  it('should purge blobs associated with attachments when purging submission', testService(async (service, { Submissions, oneFirst }) => {
+  it('should purge blobs associated with attachments when purging submission', testService(async (service, { Blobs, Submissions, oneFirst }) => {
     const asAlice = await service.login('alice');
 
     await asAlice.post('/v1/projects/1/forms?publish=true')
@@ -134,6 +134,9 @@ describe('query module submission purge', () => {
     await asAlice.delete('/v1/projects/1/forms/binaryType/submissions/both');
     await Submissions.purge(true);
 
+    // Purge unattached blobs
+    await Blobs.purgeUnattached();
+
     // One blob still remains from first submission which was not deleted
     blobCount = await oneFirst(sql`select count(*) from blobs`);
     blobCount.should.equal(1);
@@ -372,6 +375,9 @@ describe('query module submission purge', () => {
     // Purge the submission
     await container.Submissions.purge(true);
 
+    // Purge unattached blobs
+    await container.Blobs.purgeUnattached();
+
     // Check that some of the client audit events are deleted from the database
     const numClientAudits = await container.oneFirst(sql`select count(*) from client_audits`);
     numClientAudits.should.equal(3); // from the non-deleted submission