From 5bb31b0ab2cb3f3a9eec404734c0c5fbfa649fe7 Mon Sep 17 00:00:00 2001 From: Julien Loizelet Date: Fri, 9 Feb 2024 15:08:18 +0900 Subject: [PATCH 1/5] fix(sql): Add length to machine_id field for mysql compatibility --- .gitignore | 5 ++++- src/cscapi/sql_storage.py | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index b1baa0d..f805a7e 100644 --- a/.gitignore +++ b/.gitignore @@ -144,4 +144,7 @@ src/cscapi/_version.py *.db #vscode -.vscode \ No newline at end of file +.vscode + +#ddev +.ddev \ No newline at end of file diff --git a/src/cscapi/sql_storage.py b/src/cscapi/sql_storage.py index 870f6ce..39c8824 100644 --- a/src/cscapi/sql_storage.py +++ b/src/cscapi/sql_storage.py @@ -9,6 +9,7 @@ ForeignKey, Integer, TEXT, + String, create_engine, delete, update, @@ -50,7 +51,7 @@ class MachineDBModel(Base): __tablename__ = "machine_models" id = Column(Integer, primary_key=True, autoincrement=True) - machine_id = Column(TEXT, unique=True) + machine_id = Column(String(256), unique=True) token = Column(TEXT) password = Column(TEXT) scenarios = Column(TEXT) @@ -109,7 +110,7 @@ class SignalDBModel(Base): alert_id = Column(Integer, primary_key=True, autoincrement=True) created_at = Column(TEXT) - machine_id = Column(TEXT) + machine_id = Column(String(256)) scenario_version = Column(TEXT, nullable=True) message = Column(TEXT, nullable=True) uuid = Column(TEXT) From f497a6f5422d874329e344b08155d05b785900f1 Mon Sep 17 00:00:00 2001 From: Julien Loizelet Date: Fri, 9 Feb 2024 17:12:03 +0900 Subject: [PATCH 2/5] ci(test): Add workflow to test sql various engines --- .github/workflows/sql-storage.yml | 73 +++++++++++++++++++++++++++++++ default.env | 7 +-- pytest.ini | 1 + requirements-dev.txt | 5 +++ tests/test_sql_storage.py | 41 ++++++++++++++++- 5 files changed, 123 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/sql-storage.yml diff --git a/.github/workflows/sql-storage.yml b/.github/workflows/sql-storage.yml new file mode 100644 index 0000000..e166485 --- /dev/null +++ b/.github/workflows/sql-storage.yml @@ -0,0 +1,73 @@ +name: Sql storage tests + +on: + push: + branches: [ main, feat/add-test-for-sql-database-types ] + paths-ignore: + - '**.md' + pull_request: + branches: [ main ] + paths-ignore: + - '**.md' + workflow_dispatch: + +jobs: + sql-tests: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + sql-engine: ["sqlite", "mysql:5.5", "mysql:5.7", "mysql:8.0", "postgres:9", "postgres:16", "mariadb:10.0", "mariadb:10.8"] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Set engine code + run: echo "ENGINE_CODE=$(echo ${{ matrix.sql-engine }} | sed 's/:.*//')" >> $GITHUB_ENV + + - name: Install DDEV + if: ${{ matrix.sql-engine != 'sqlite' }} + run: | + # @see https://ddev.readthedocs.io/en/stable/#installationupgrade-script-linux-and-macos-armarm64-and-amd64-architectures + curl -fsSL https://apt.fury.io/drud/gpg.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/ddev.gpg > /dev/null + echo "deb [signed-by=/etc/apt/trusted.gpg.d/ddev.gpg] https://apt.fury.io/drud/ * *" | sudo tee /etc/apt/sources.list.d/ddev.list + sudo apt-get -q update + sudo apt-get -q -y install libnss3-tools ddev + mkcert -install + ddev config global --instrumentation-opt-in=false --omit-containers=ddev-ssh-agent + + - name: Create DDEV project + if: ${{ matrix.sql-engine != 'sqlite' }} + run: | + ddev config --project-type=python --database ${{ matrix.sql-engine }} --project-name=crowdsec-python-capi-sdk --webserver-type nginx-fpm --host-db-port 5432 + ddev start + + - name: Grant privileges + if: contains(fromJson('["mysql","mariadb"]'),env.ENGINE_CODE) + run: | + ddev mysql -uroot -proot -e "GRANT ALL PRIVILEGES ON *.* to 'db'@'%';" + + - name: Set .env variable + run: | + echo "TEST_SQL_ENGINE=${{ env.ENGINE_CODE }}" > .env + + - name: Install setuptools + if: contains(fromJson('["3.12"]'),matrix.python-version) + run: | + python -m pip install --upgrade pip setuptools wheel + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python setup.py install + python -m pip install -r requirements-dev.txt + + - name: Tests + run: | + python -m pytest tests/test_sql_storage.py -s diff --git a/default.env b/default.env index a5f3498..5667cf1 100644 --- a/default.env +++ b/default.env @@ -1,3 +1,4 @@ -CROWDSEC_TEST_VERSION="dev" -CROWDSEC_TEST_FLAVORS="full" -CROWDSEC_TEST_NETWORK="net-test" +TEST_SQL_ENGINE="sqlite" +TEST_POSTGRESQL_URL="postgresql+pg8000://db:db@localhost:5432/" +TEST_MYSQL_URL="mysql+mysqlconnector://db:db@localhost:5432/" +TEST_MARIADB_URL="mariadb+pymysql://db:db@localhost:5432/" \ No newline at end of file diff --git a/pytest.ini b/pytest.ini index 8147d41..f4d0f38 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,5 @@ [pytest] +timeout = 60 addopts = --pdbcls=IPython.terminal.debugger:Pdb --ignore=test/install diff --git a/requirements-dev.txt b/requirements-dev.txt index 2a25a6d..063c2b0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,4 +2,9 @@ black pytest pytest-dotenv pytest-httpx +pytest-timeout +sqlalchemy-utils +mysql-connector-python +pymysql +pg8000 diff --git a/tests/test_sql_storage.py b/tests/test_sql_storage.py index ac35ce9..b2b6486 100644 --- a/tests/test_sql_storage.py +++ b/tests/test_sql_storage.py @@ -12,16 +12,55 @@ ) from cscapi.storage import MachineModel, SourceModel +from sqlalchemy import ( + create_engine, +) +from sqlalchemy_utils import database_exists, create_database, drop_database +from sqlalchemy.pool import NullPool + from .test_client import mock_signals class TestSQLStorage(TestCase): def setUp(self) -> None: self.db_path = f"{str(int(time.time()))}.db" - db_uri = f"sqlite:///{self.db_path}" + # Use .env file to modify variables + engine_type = ( + os.getenv("TEST_SQL_ENGINE") if os.getenv("TEST_SQL_ENGINE") else "sqlite" + ) + if engine_type == "sqlite": + db_uri = f"sqlite:///{self.db_path}" + elif engine_type == "postgres": + db_uri = f"{os.getenv('TEST_POSTGRESQL_URL')}{self.db_path}" + elif engine_type == "mysql": + db_uri = f"{os.getenv('TEST_MYSQL_URL')}{self.db_path}" + elif engine_type == "mariadb": + db_uri = f"{os.getenv('TEST_MARIADB_URL')}{self.db_path}" + else: + raise ValueError(f"Unknown engine type: {engine_type}") + if not database_exists(db_uri): + create_database(db_uri) + self.storage: SQLStorage = SQLStorage(db_uri) + self.db_uri = db_uri + print(f"Using {engine_type} engine with {db_uri}") def tearDown(self) -> None: + # postgresql, mysql, mariadb + if database_exists(self.db_uri): + if self.storage.session: + self.storage.session.close() + engine = create_engine(self.db_uri, poolclass=NullPool) + conn = engine.connect() + try: + drop_database(self.db_uri) + except Exception as e: + print(f"Error occurred while dropping the database: {e}") + + conn.close() + engine.dispose() + + # sqlite try: os.remove(self.db_path) except: From 7baef73e95dec01d83e3fca4c3418744d807fdb6 Mon Sep 17 00:00:00 2001 From: Julien Loizelet Date: Fri, 9 Feb 2024 18:18:02 +0900 Subject: [PATCH 3/5] fix(mysql): Decrease machine_id length for mysql 5.6 and prior versions --- src/cscapi/sql_storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cscapi/sql_storage.py b/src/cscapi/sql_storage.py index 39c8824..12eb485 100644 --- a/src/cscapi/sql_storage.py +++ b/src/cscapi/sql_storage.py @@ -51,7 +51,7 @@ class MachineDBModel(Base): __tablename__ = "machine_models" id = Column(Integer, primary_key=True, autoincrement=True) - machine_id = Column(String(256), unique=True) + machine_id = Column(String(191), unique=True) token = Column(TEXT) password = Column(TEXT) scenarios = Column(TEXT) @@ -110,7 +110,7 @@ class SignalDBModel(Base): alert_id = Column(Integer, primary_key=True, autoincrement=True) created_at = Column(TEXT) - machine_id = Column(String(256)) + machine_id = Column(String(191)) scenario_version = Column(TEXT, nullable=True) message = Column(TEXT, nullable=True) uuid = Column(TEXT) From 763a326e1a9164f10cf55329b526e3b42b54c185 Mon Sep 17 00:00:00 2001 From: Julien Loizelet Date: Fri, 9 Feb 2024 18:35:57 +0900 Subject: [PATCH 4/5] ci(test sql): Restrict workflow branch trigger to main --- .github/workflows/sql-storage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sql-storage.yml b/.github/workflows/sql-storage.yml index e166485..e28f585 100644 --- a/.github/workflows/sql-storage.yml +++ b/.github/workflows/sql-storage.yml @@ -2,7 +2,7 @@ name: Sql storage tests on: push: - branches: [ main, feat/add-test-for-sql-database-types ] + branches: [ main ] paths-ignore: - '**.md' pull_request: From 87490f2290f8bd9befa65e7ff3674b700368bc24 Mon Sep 17 00:00:00 2001 From: Julien Loizelet Date: Fri, 9 Feb 2024 19:07:22 +0900 Subject: [PATCH 5/5] feat(sql): Change machine_id length to 128 --- src/cscapi/sql_storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cscapi/sql_storage.py b/src/cscapi/sql_storage.py index 12eb485..7cbb8bb 100644 --- a/src/cscapi/sql_storage.py +++ b/src/cscapi/sql_storage.py @@ -51,7 +51,7 @@ class MachineDBModel(Base): __tablename__ = "machine_models" id = Column(Integer, primary_key=True, autoincrement=True) - machine_id = Column(String(191), unique=True) + machine_id = Column(String(128), unique=True) token = Column(TEXT) password = Column(TEXT) scenarios = Column(TEXT) @@ -110,7 +110,7 @@ class SignalDBModel(Base): alert_id = Column(Integer, primary_key=True, autoincrement=True) created_at = Column(TEXT) - machine_id = Column(String(191)) + machine_id = Column(String(128)) scenario_version = Column(TEXT, nullable=True) message = Column(TEXT, nullable=True) uuid = Column(TEXT)