Initial Push for Batch Deletes

utopia-php · Sep 10, 2024 · 170f8d5 · 170f8d5
1 parent 87fa42f
commit 170f8d5
Show file tree

Hide file tree

Showing 6 changed files with 356 additions and 0 deletions.
diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php
@@ -587,6 +587,16 @@ abstract public function updateDocuments(string $collection, array $documents, i
      */
     abstract public function deleteDocument(string $collection, string $id): bool;
 
+    /**
+     * Delete Documents
+     *
+     * @param string $collection
+     * @param array<\Utopia\Database\Query> $queries
+     *
+     * @return bool
+     */
+    abstract public function deleteDocuments(string $collection, array $queries): bool;
+
     /**
      * Find Documents
      *

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
@@ -1635,6 +1635,61 @@ public function deleteDocument(string $collection, string $id): bool
         return $deleted;
     }
 
+    /**
+     * Delete Documents
+     *
+     * @param string $collection
+     * @param array<\Utopia\Database\Query> $queries
+     *
+     * @return bool
+     */
+    public function deleteDocuments(string $collection, array $queries): bool
+    {
+        $name = $this->filter($collection);
+        $where = [];
+
+        $queries = array_map(fn ($query) => clone $query, $queries);
+
+        $conditions = $this->getSQLConditions($queries);
+        if(!empty($conditions)) {
+            $where[] = $conditions;
+        }
+
+        if ($this->sharedTables) {
+            $where[] = "table_main._tenant = :_tenant";
+        }
+
+        $sqlWhere = !empty($where) ? 'WHERE ' . implode(' AND ', $where) : '';
+
+        $selections = $this->getAttributeSelections($queries);
+
+        $sql = "
+            USE {$this->database};
+            DELETE {$this->getAttributeProjection($selections, 'table_main')}
+            FROM {$this->getSQLTable($name)} as table_main
+            {$sqlWhere};
+        ";
+
+        $sql = $this->trigger(Database::EVENT_DOCUMENT_DELETE, $sql);
+
+        $stmt = $this->getPDO()->prepare($sql);
+
+        foreach ($queries as $query) {
+            $this->bindConditionValue($stmt, $query);
+        }
+        if ($this->sharedTables) {
+            $stmt->bindValue(':_tenant', $this->tenant);
+        }
+
+        try {
+            $stmt->execute();
+        } catch (PDOException $e) {
+            $this->processException($e);
+        }
+
+        return true;
+    }
+
     /**
      * Find Documents
      *

diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php
@@ -909,6 +909,19 @@ public function deleteDocument(string $collection, string $id): bool
         return (!!$result);
     }
 
+    /**
+     * Delete Documents
+     *
+     * @param string $collection
+     * @param array<\Utopia\Database\Query> $queries
+     *
+     * @return bool
+     */
+    public function deleteDocuments(string $collection, array $queries): bool
+    {
+        throw new \Exception('Not Implemented');
+    }
+
     /**
      * Update Attribute.
      * @param string $collection

diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php
@@ -1558,6 +1558,61 @@ public function deleteDocument(string $collection, string $id): bool
         return $deleted;
     }
 
+
+    /**
+     * Delete Documents
+     *
+     * @param string $collection
+     * @param array<\Utopia\Database\Query> $queries
+     *
+     * @return bool
+     */
+    public function deleteDocuments(string $collection, array $queries): bool
+    {
+        $name = $this->filter($collection);
+        $where = [];
+
+        $queries = array_map(fn ($query) => clone $query, $queries);
+
+        $conditions = $this->getSQLConditions($queries);
+        if(!empty($conditions)) {
+            $where[] = $conditions;
+        }
+
+        if ($this->sharedTables) {
+            $where[] = "table_main._tenant = :_tenant";
+        }
+
+        $sqlWhere = !empty($where) ? 'WHERE ' . implode(' AND ', $where) : '';
+
+        $selections = $this->getAttributeSelections($queries);
+
+        $sql = "
+            SELECT {$this->getAttributeProjection($selections, 'table_main')}
+            FROM {$this->getSQLTable($name)} as table_main
+            {$sqlWhere}
+        ";
+
+        $sql = $this->trigger(Database::EVENT_DOCUMENT_DELETE, $sql);
+
+        $stmt = $this->getPDO()->prepare($sql);
+
+        foreach ($queries as $query) {
+            $this->bindConditionValue($stmt, $query);
+        }
+        if ($this->sharedTables) {
+            $stmt->bindValue(':_tenant', $this->tenant);
+        }
+
+        try {
+            $stmt->execute();
+        } catch (PDOException $e) {
+            $this->processException($e);
+        }
+
+        return true;
+    }
+
     /**
      * Find Documents
      *

diff --git a/src/Database/Database.php b/src/Database/Database.php
@@ -4944,6 +4944,84 @@ private function deleteCascade(Document $collection, Document $relatedCollection
         }
     }
 
+    /**
+     * Delete Documents
+     *
+     * Deletes all documents which match the given query, will respect the relationship's onDelete optin.
+     *
+     * @param string $collection
+     * @param array<Query> $queries
+     *
+     * @return bool
+     *
+     * @throws AuthorizationException
+     * @throws DatabaseException
+     * @throws RestrictedException
+     */
+    public function deleteDocuments(string $collection, array $queries = []): bool
+    {
+        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
+            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
+        }
+
+        $collection = $this->silent(fn () => $this->getCollection($collection));
+
+        $deleted = $this->withTransaction(function () use ($collection, $queries) {
+            $lastDocument = null;
+            while (true) {
+                $affectedDocuments = $this->find($collection->getId(), array_merge(
+                    empty($lastDocument) ? [
+                        Query::limit(100),
+                    ] : [
+                        Query::limit(100),
+                        Query::cursorAfter($lastDocument),
+                    ],
+                    $queries,
+                ));
+
+                if (empty($affectedDocuments)) {
+                    break;
+                }
+
+                foreach ($affectedDocuments as $document) {
+                    $validator = new Authorization(self::PERMISSION_DELETE);
+
+                    if ($collection->getId() !== self::METADATA) {
+                        $documentSecurity = $collection->getAttribute('documentSecurity', false);
+                        if (!$validator->isValid([
+                            ...$collection->getDelete(),
+                            ...($documentSecurity ? $document->getDelete() : [])
+                        ])) {
+                            throw new AuthorizationException($validator->getDescription());
+                        }
+                    }
+
+                    // Delete Relationships
+                    if ($this->resolveRelationships) {
+                        $document = $this->silent(fn () => $this->deleteDocumentRelationships($collection, $document));
+                    }
+
+                    // Fire events
+                    $this->trigger(self::EVENT_DOCUMENT_DELETE, $document);
+
+                    $this->purgeRelatedDocuments($collection, $document->getId());
+                    $this->purgeCachedDocument($collection->getId(), $document->getId());
+                }
+
+                if (count($affectedDocuments) < 100) {
+                    break;
+                } else {
+                    $lastDocument = end($affectedDocuments);
+                }
+            }
+
+            // Mass delete using adapter with query
+            return $this->adapter->deleteDocuments($collection->getId(), $queries);
+        });
+
+        return $deleted;
+    }
+
     /**
      * Cleans the all the collection's documents from the cache
      * And the all related cached documents.

diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php
@@ -15608,4 +15608,149 @@ public function testEvents(): void
             $database->delete('hellodb');
         });
     }
+
+    public function propegateBulkDocuments(bool $documentSecurity = false): void
+    {
+        for ($i = 0; $i < 10; $i++) {
+            var_dump(new Document(
+                array_merge([
+                    '$id' => 'doc' . $i,
+                    'text' => 'value' . $i,
+                    'integer' => $i
+                ], $documentSecurity ? [
+                    '$permissions' => [
+                        Permission::delete(Role::any()),
+                        Permission::create(Role::any()),
+                        Permission::read(Role::any()),
+                    ],
+                ] : [])
+            ));
+
+            static::getDatabase()->createDocument('bulk_delete', new Document(
+                array_merge([
+                    '$id' => 'doc' . $i,
+                    'text' => 'value' . $i,
+                    'integer' => $i
+                ], $documentSecurity ? [
+                    '$permissions' => [
+                        Permission::create(Role::any()),
+                        Permission::read(Role::any()),
+                    ],
+                ] : [])
+            ));
+        }
+    }
+
+    public function testDeleteBulkDocuments(): void
+    {
+        static::getDatabase()->createCollection(
+            'bulk_delete',
+            attributes: [
+                new Document([
+                    '$id' => 'text',
+                    'type' => Database::VAR_STRING,
+                    'size' => 100,
+                    'required' => true,
+                ]),
+                new Document([
+                    '$id' => 'integer',
+                    'type' => Database::VAR_INTEGER,
+                    'size' => 10,
+                    'required' => true,
+                ])
+            ],
+            documentSecurity: false,
+            permissions: [
+                Permission::create(Role::any()),
+                Permission::read(Role::any()),
+                Permission::delete(Role::any())
+            ]
+        );
+
+        $this->propegateBulkDocuments();
+
+        $docs = static::getDatabase()->find('bulk_delete');
+        $this->assertCount(10, $docs);
+
+        // TEST: Bulk Delete All Documents
+        static::getDatabase()->deleteDocuments('bulk_delete');
+
+        $docs = static::getDatabase()->find('bulk_delete');
+        $this->assertCount(0, $docs);
+
+        // TEST: Bulk delete documents with queries.
+        $this->propegateBulkDocuments();
+
+        static::getDatabase()->deleteDocuments('bulk_delete', [
+            Query::greaterThanEqual('integer', 5)
+        ]);
+
+        $docs = static::getDatabase()->find('bulk_delete');
+        $this->assertCount(5, $docs);
+
+        // TEST (FAIL): Bulk delete all documents with invalid collection permission
+        static::getDatabase()->updateCollection('bulk_delete', [], false);
+        try {
+            static::getDatabase()->deleteDocuments('bulk_delete');
+            $this->fail('Bulk deleted documents with invalid collection permission');
+        } catch (\Utopia\Database\Exception\Authorization) {
+        }
+
+        static::getDatabase()->updateCollection('bulk_delete', [
+            Permission::create(Role::any()),
+            Permission::read(Role::any()),
+            Permission::delete(Role::any())
+        ], false);
+        static::getDatabase()->deleteDocuments('bulk_delete');
+        $this->assertEquals(0, count($this->getDatabase()->find('bulk_delete')));
+
+        // TEST (FAIL): Bulk delete all documents with invalid document permissions
+        // Authorization::setRole(Role::any()->toString());
+        static::getDatabase()->updateCollection('bulk_delete', [
+            Permission::create(Role::any()),
+        ], true);
+        $this->propegateBulkDocuments(true);
+
+        try {
+            static::getDatabase()->deleteDocuments('bulk_delete');
+            $this->fail('Bulk deleted documents with invalid document permission');
+        } catch (\Utopia\Database\Exception\Authorization) {
+        }
+
+        static::getDatabase()->updateCollection('bulk_delete', [
+            Permission::create(Role::any()),
+            Permission::read(Role::any()),
+            Permission::delete(Role::any())
+        ], false);
+        static::getDatabase()->deleteDocuments('bulk_delete');
+        $this->assertEquals(0, count($this->getDatabase()->find('bulk_delete')));
+
+        // Teardown
+        static::getDatabase()->deleteCollection('bulk_delete');
+    }
+
+
+    public function testDeleteBulkDocumentsRelationships(): void
+    {
+        if (!static::getDatabase()->getAdapter()->getSupportForRelationships()) {
+            $this->expectNotToPerformAssertions();
+            return;
+        }
+
+        static::getDatabase()->createCollection('bulk_delete_r_1');
+        static::getDatabase()->createCollection('bulk_delete_r_2');
+
+        // ONE_TO_ONE
+        static::getDatabase()->createRelationship(
+            collection: 'bulk_delete_r_1',
+            relatedCollection: 'bulk_delete_r_2',
+            type: Database::RELATION_ONE_TO_ONE,
+        );
+
+        // Restrict
+
+        // NULL
+
+        // Cascade
+    }
 }