Merge pull request #272 from faisalill/main

Add a function to get the size of a collection/table in bytes.
utopia-php · Aug 10, 2023 · 318ee12 · 318ee12
2 parents 317d6a1 + 3b65416
commit 318ee12
Show file tree

Hide file tree

Showing 8 changed files with 272 additions and 0 deletions.
diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php
@@ -400,6 +400,15 @@ abstract public function sum(string $collection, string $attribute, array $queri
      */
     abstract public function count(string $collection, array $queries = [], ?int $max = null, ?int $timeout = null): int;
 
+    /**
+     * Get Collection Size
+     *
+     * @param string $collection
+     * @return int
+     * @throws DatabaseException
+     */
+    abstract public function getSizeOfCollection(string $collection): int;
+
     /**
      * Get max STRING limit
      *

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
@@ -142,6 +142,46 @@ public function createCollection(string $name, array $attributes = [], array $in
         return true;
     }
 
+    /**
+     * Get Collection Size
+     * @param string $collection
+     * @return int
+     * @throws DatabaseException
+     */
+    public function getSizeOfCollection(string $collection): int
+    {
+        $collection = $this->filter($collection);
+        $collection = $this->getNamespace() . '_' . $collection;
+        $database = $this->getDefaultDatabase();
+        $name = $database . '/' . $collection;
+        $permissions = $database . '/' . $collection . '_perms';
+
+        $collectionSize = $this->getPDO()->prepare("
+            SELECT SUM(FS_BLOCK_SIZE + ALLOCATED_SIZE)  
+            FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESPACES
+            WHERE NAME = :name
+         ");
+
+        $permissionsSize = $this->getPDO()->prepare("
+            SELECT SUM(FS_BLOCK_SIZE + ALLOCATED_SIZE)  
+            FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESPACES
+            WHERE NAME = :permissions
+        ");
+
+        $collectionSize->bindParam(':name', $name);
+        $permissionsSize->bindParam(':permissions', $permissions);
+
+        try {
+            $collectionSize->execute();
+            $permissionsSize->execute();
+            $size = $collectionSize->fetchColumn() + $permissionsSize->fetchColumn();
+        } catch (PDOException $e) {
+            throw new DatabaseException('Failed to get collection size: ' . $e->getMessage());
+        }
+
+        return $size;
+    }
+
     /**
      * Delete Collection
      * @param string $id

diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php
@@ -253,6 +253,35 @@ public function listCollections(): array
         return $list;
     }
 
+    /**
+     * Get Collection Size
+     * @param string $collection
+     * @return int
+     * @throws DatabaseException
+     */
+    public function getSizeOfCollection(string $collection): int
+    {
+        $namespace = $this->getNamespace();
+        $collection = $this->filter($collection);
+        $collection = $namespace. '_' . $collection;
+
+        $command = [
+            'collStats' => $collection,
+            'scale' => 1
+        ];
+
+        try {
+            $result = $this->getClient()->query($command);
+            if (is_object($result)) {
+                return $result->totalSize;
+            } else {
+                throw new DatabaseException('No size found');
+            }
+        } catch(Exception $e) {
+            throw new DatabaseException('Failed to get collection size: ' . $e->getMessage());
+        }
+    }
+
     /**
      * Delete Collection
      *

diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php
@@ -78,4 +78,44 @@ protected function processException(PDOException $e): void
 
         throw $e;
     }
+
+     /**
+     * Get Collection Size
+     * @param string $collection
+     * @return int
+     * @throws DatabaseException
+     */
+    public function getSizeOfCollection(string $collection): int
+    {
+        $collection = $this->filter($collection);
+        $collection = $this->getNamespace() . '_' . $collection;
+        $database = $this->getDefaultDatabase();
+        $name = $database . '/' . $collection;
+        $permissions = $database . '/' . $collection . '_perms';
+
+        $collectionSize = $this->getPDO()->prepare("
+             SELECT SUM(FS_BLOCK_SIZE + ALLOCATED_SIZE)  
+             FROM INFORMATION_SCHEMA.INNODB_TABLESPACES
+             WHERE NAME = :name
+        ");
+
+        $permissionsSize = $this->getPDO()->prepare("
+             SELECT SUM(FS_BLOCK_SIZE + ALLOCATED_SIZE)  
+             FROM INFORMATION_SCHEMA.INNODB_TABLESPACES
+             WHERE NAME = :permissions
+        ");
+
+        $collectionSize->bindParam(':name', $name);
+        $permissionsSize->bindParam(':permissions', $permissions);
+
+        try {
+            $collectionSize->execute();
+            $permissionsSize->execute();
+            $size = $collectionSize->fetchColumn() + $permissionsSize->fetchColumn();
+        } catch (PDOException $e) {
+            throw new DatabaseException('Failed to get collection size: ' . $e->getMessage());
+        }
+
+        return $size;
+    }
 }
diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php
@@ -151,6 +151,41 @@ public function createCollection(string $name, array $attributes = [], array $in
         return true;
     }
 
+    /**
+     * Get Collection Size
+     * @param string $collection
+     * @return int
+     * @throws DatabaseException
+     *
+     */
+    public function getSizeOfCollection(string $collection): int
+    {
+        $collection = $this->filter($collection);
+        $name = $this->getSQLTable($collection);
+        $permissions = $this->getSQLTable($collection . '_perms');
+
+        $collectionSize = $this->getPDO()->prepare("
+             SELECT pg_total_relation_size(:name);
+        ");
+
+        $permissionsSize = $this->getPDO()->prepare("
+             SELECT pg_total_relation_size(:permissions);
+        ");
+
+        $collectionSize->bindParam(':name', $name);
+        $permissionsSize->bindParam(':permissions', $permissions);
+
+        try {
+            $collectionSize->execute();
+            $permissionsSize->execute();
+            $size = $collectionSize->fetchColumn() + $permissionsSize->fetchColumn();
+        } catch (PDOException $e) {
+            throw new DatabaseException('Failed to get collection size: ' . $e->getMessage());
+        }
+
+        return  $size;
+    }
+
     /**
      * Delete Collection
      *

diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php
@@ -158,6 +158,42 @@ public function createCollection(string $name, array $attributes = [], array $in
         return true;
     }
 
+    /**
+     * Get Collection Size
+     * @param string $collection
+     * @return int
+     * @throws DatabaseException
+     *
+     */
+    public function getSizeOfCollection(string $collection): int
+    {
+        $collection = $this->filter($collection);
+        $namespace = $this->getNamespace();
+        $name = $namespace . '_' . $collection;
+        $permissions = $namespace . '_' . $collection . '_perms';
+
+        $collectionSize = $this->getPDO()->prepare("
+             SELECT SUM(\"pgsize\") FROM \"dbstat\" WHERE name=:name;
+        ");
+
+        $permissionsSize = $this->getPDO()->prepare("
+             SELECT SUM(\"pgsize\") FROM \"dbstat\" WHERE name=:name;
+        ");
+
+        $collectionSize->bindParam(':name', $name);
+        $permissionsSize->bindParam(':name', $permissions);
+
+        try {
+            $collectionSize->execute();
+            $permissionsSize->execute();
+            $size = $collectionSize->fetchColumn() + $permissionsSize->fetchColumn();
+        } catch (PDOException $e) {
+            throw new DatabaseException('Failed to get collection size: ' . $e->getMessage());
+        }
+
+        return $size;
+    }
+
     /**
      * Delete Collection
      * @param string $id

diff --git a/src/Database/Database.php b/src/Database/Database.php
@@ -803,6 +803,18 @@ public function listCollections(int $limit = 25, int $offset = 0): array
         return $result;
     }
 
+    /**
+     * Get Collection Size
+     *
+     * @param string $collection
+     *
+     * @return int
+     */
+    public function getSizeOfCollection(string $collection): int
+    {
+        return $this->adapter->getSizeOfCollection($collection);
+    }
+
     /**
      * Delete Collection
      *

diff --git a/tests/Database/Base.php b/tests/Database/Base.php
@@ -280,6 +280,77 @@ public function testCreateListExistsDeleteCollection(): void
         $this->assertEquals(false, static::getDatabase()->exists($this->testDatabase, 'actors'));
     }
 
+    public function testSizeCollection(): void
+    {
+        static::getDatabase()->createCollection('sizeTest1');
+        static::getDatabase()->createCollection('sizeTest2');
+
+        $size1 = static::getDatabase()->getSizeOfCollection('sizeTest1');
+        $size2 = static::getDatabase()->getSizeOfCollection('sizeTest2');
+        $sizeDifference = abs($size1 - $size2);
+        // Size of an empty collection returns either 172032 or 167936 bytes randomly
+        // Therefore asserting with a tolerance of 5000 bytes
+        $byteDifference = 5000;
+        $this->assertLessThan($byteDifference, $sizeDifference);
+
+        static::getDatabase()->createAttribute('sizeTest2', 'string1', Database::VAR_STRING, 128, true);
+        static::getDatabase()->createAttribute('sizeTest2', 'string2', Database::VAR_STRING, 254 + 1, true);
+        static::getDatabase()->createAttribute('sizeTest2', 'string3', Database::VAR_STRING, 254 + 1, true);
+        static::getDatabase()->createIndex('sizeTest2', 'index', Database::INDEX_KEY, ['string1', 'string2', 'string3'], [128, 128, 128]);
+
+        $loopCount = 40;
+
+        for ($i = 0; $i < $loopCount; $i++) {
+            static::getDatabase()->createDocument('sizeTest2', new Document([
+                'string1' => 'string1' . $i,
+                'string2' => 'string2' . $i,
+                'string3' => 'string3' . $i,
+            ]));
+        }
+
+        $size2 = static::getDatabase()->getSizeOfCollection('sizeTest2');
+
+        $this->assertGreaterThan($size1, $size2);
+    }
+
+    public function testSizeFullText(): void
+    {
+        // SQLite does not support fulltext indexes
+        if (!static::getDatabase()->getAdapter()->getSupportForFulltextIndex()) {
+            $this->expectNotToPerformAssertions();
+            return;
+        }
+
+        static::getDatabase()->createCollection('fullTextSizeTest');
+
+        $size1 = static::getDatabase()->getSizeOfCollection('fullTextSizeTest');
+
+        static::getDatabase()->createAttribute('fullTextSizeTest', 'string1', Database::VAR_STRING, 128, true);
+        static::getDatabase()->createAttribute('fullTextSizeTest', 'string2', Database::VAR_STRING, 254, true);
+        static::getDatabase()->createAttribute('fullTextSizeTest', 'string3', Database::VAR_STRING, 254, true);
+        static::getDatabase()->createIndex('fullTextSizeTest', 'index', Database::INDEX_KEY, ['string1', 'string2', 'string3'], [128, 128, 128]);
+
+        $loopCount = 10;
+
+        for ($i = 0; $i < $loopCount; $i++) {
+            static::getDatabase()->createDocument('fullTextSizeTest', new Document([
+                'string1' => 'string1' . $i,
+                'string2' => 'string2' . $i,
+                'string3' => 'string3' . $i,
+            ]));
+        }
+
+        $size2 = static::getDatabase()->getSizeOfCollection('fullTextSizeTest');
+
+        $this->assertGreaterThan($size1, $size2);
+
+        static::getDatabase()->createIndex('fullTextSizeTest', 'fulltext_index', Database::INDEX_FULLTEXT, ['string1']);
+
+        $size3 = static::getDatabase()->getSizeOfCollection('fullTextSizeTest');
+
+        $this->assertGreaterThan($size2, $size3);
+    }
+
     public function testCreateDeleteAttribute(): void
     {
         static::getDatabase()->createCollection('attributes');