fix(sql): Decrease machine_id length for mysql 5.6 and prior versions (…

…crowdsecurity#18)

* fix(sql): Add length to machine_id field for mysql compatibility

* ci(test): Add workflow to test sql various engines

* fix(mysql): Decrease machine_id length for mysql 5.6 and prior versions

* ci(test sql): Restrict workflow branch trigger to main

* feat(sql): Change machine_id length to 128

.github/workflows/sql-storage.yml

@@ -0,0 +1,73 @@
+name: Sql storage tests
+
+on:
+  push:
+    branches: [ main ]
+    paths-ignore:
+      - '**.md'
+  pull_request:
+    branches: [ main ]
+    paths-ignore:
+      - '**.md'
+  workflow_dispatch:
+
+jobs:
+  sql-tests:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12"]
+        sql-engine: ["sqlite", "mysql:5.5", "mysql:5.7", "mysql:8.0", "postgres:9", "postgres:16", "mariadb:10.0", "mariadb:10.8"]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Set engine code
+      run: echo "ENGINE_CODE=$(echo ${{ matrix.sql-engine }} | sed 's/:.*//')" >> $GITHUB_ENV
+
+    - name: Install DDEV
+      if: ${{ matrix.sql-engine != 'sqlite' }}
+      run: |
+        # @see https://ddev.readthedocs.io/en/stable/#installationupgrade-script-linux-and-macos-armarm64-and-amd64-architectures
+        curl -fsSL https://apt.fury.io/drud/gpg.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/ddev.gpg > /dev/null
+        echo "deb [signed-by=/etc/apt/trusted.gpg.d/ddev.gpg] https://apt.fury.io/drud/ * *" | sudo tee /etc/apt/sources.list.d/ddev.list
+        sudo apt-get -q update
+        sudo apt-get -q -y install libnss3-tools ddev
+        mkcert -install
+        ddev config global --instrumentation-opt-in=false --omit-containers=ddev-ssh-agent
+
+    - name: Create DDEV project
+      if: ${{ matrix.sql-engine != 'sqlite' }}
+      run: |
+        ddev config --project-type=python --database ${{ matrix.sql-engine }} --project-name=crowdsec-python-capi-sdk --webserver-type nginx-fpm --host-db-port 5432
+        ddev start
+
+    - name: Grant privileges
+      if: contains(fromJson('["mysql","mariadb"]'),env.ENGINE_CODE)
+      run: |
+        ddev mysql -uroot -proot -e "GRANT ALL PRIVILEGES ON *.* to 'db'@'%';"   
+
+    - name: Set .env variable
+      run: |
+        echo "TEST_SQL_ENGINE=${{ env.ENGINE_CODE }}" > .env
+
+    - name: Install setuptools
+      if: contains(fromJson('["3.12"]'),matrix.python-version)
+      run: |
+        python -m pip install --upgrade pip setuptools wheel    
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python setup.py install
+        python -m pip install -r requirements-dev.txt
+
+    - name: Tests
+      run: |
+        python -m pytest tests/test_sql_storage.py -s

default.env

@@ -1,3 +1,4 @@
-CROWDSEC_TEST_VERSION="dev"
-CROWDSEC_TEST_FLAVORS="full"
-CROWDSEC_TEST_NETWORK="net-test"
+TEST_SQL_ENGINE="sqlite"
+TEST_POSTGRESQL_URL="postgresql+pg8000://db:db@localhost:5432/"
+TEST_MYSQL_URL="mysql+mysqlconnector://db:db@localhost:5432/"
+TEST_MARIADB_URL="mariadb+pymysql://db:db@localhost:5432/"

pytest.ini

@@ -1,4 +1,5 @@
 [pytest]
+timeout = 60
 addopts =
     --pdbcls=IPython.terminal.debugger:Pdb
     --ignore=test/install

requirements-dev.txt

@@ -2,4 +2,9 @@ black
 pytest
 pytest-dotenv
 pytest-httpx
+pytest-timeout
+sqlalchemy-utils
+mysql-connector-python
+pymysql
+pg8000
 

src/cscapi/sql_storage.py

@@ -51,7 +51,7 @@ class MachineDBModel(Base):
     __tablename__ = "machine_models"
 
     id = Column(Integer, primary_key=True, autoincrement=True)
-    machine_id = Column(String(256), unique=True)
+    machine_id = Column(String(128), unique=True)
     token = Column(TEXT)
     password = Column(TEXT)
     scenarios = Column(TEXT)
@@ -110,7 +110,7 @@ class SignalDBModel(Base):
 
     alert_id = Column(Integer, primary_key=True, autoincrement=True)
     created_at = Column(TEXT)
-    machine_id = Column(String(256))
+    machine_id = Column(String(128))
     scenario_version = Column(TEXT, nullable=True)
     message = Column(TEXT, nullable=True)
     uuid = Column(TEXT)

tests/test_sql_storage.py

@@ -12,16 +12,55 @@
 )
 from cscapi.storage import MachineModel, SourceModel
 
+from sqlalchemy import (
+    create_engine,
+)
+from sqlalchemy_utils import database_exists, create_database, drop_database
+from sqlalchemy.pool import NullPool
+
 from .test_client import mock_signals
 
 
 class TestSQLStorage(TestCase):
     def setUp(self) -> None:
         self.db_path = f"{str(int(time.time()))}.db"
-        db_uri = f"sqlite:///{self.db_path}"
+        # Use .env file to modify variables
+        engine_type = (
+            os.getenv("TEST_SQL_ENGINE") if os.getenv("TEST_SQL_ENGINE") else "sqlite"
+        )
+        if engine_type == "sqlite":
+            db_uri = f"sqlite:///{self.db_path}"
+        elif engine_type == "postgres":
+            db_uri = f"{os.getenv('TEST_POSTGRESQL_URL')}{self.db_path}"
+        elif engine_type == "mysql":
+            db_uri = f"{os.getenv('TEST_MYSQL_URL')}{self.db_path}"
+        elif engine_type == "mariadb":
+            db_uri = f"{os.getenv('TEST_MARIADB_URL')}{self.db_path}"
+        else:
+            raise ValueError(f"Unknown engine type: {engine_type}")
+        if not database_exists(db_uri):
+            create_database(db_uri)
+
         self.storage: SQLStorage = SQLStorage(db_uri)
+        self.db_uri = db_uri
+        print(f"Using {engine_type} engine with {db_uri}")
 
     def tearDown(self) -> None:
+        # postgresql, mysql, mariadb
+        if database_exists(self.db_uri):
+            if self.storage.session:
+                self.storage.session.close()
+            engine = create_engine(self.db_uri, poolclass=NullPool)
+            conn = engine.connect()
+            try:
+                drop_database(self.db_uri)
+            except Exception as e:
+                print(f"Error occurred while dropping the database: {e}")
+
+            conn.close()
+            engine.dispose()
+
+        # sqlite
         try:
             os.remove(self.db_path)
         except: