Merge pull request #13 from certego/ci_upgrade

Ci upgrade and linters

.github/.pre-commit-config.yaml

@@ -0,0 +1,19 @@
+repos:
+-   repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
+    hooks:
+    - id: flake8
+      exclude: docs/conf.py
+      args: ["--config", ".github/configurations/python_linters/.flake8"]
+
+-   repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        args: ["--settings-path", ".github/configurations/python_linters/.isort.cfg", "--filter-files", "--skip", "venv"]
+
+-   repo: https://github.com/psf/black
+    rev: 23.9.1
+    hooks:
+      - id: black
+        args: ["--config", ".github/configurations/python_linters/.black"]

.github/configurations/python_linters/.black

@@ -0,0 +1,8 @@
+[tool.black]
+line-length=160
+extend-exclude='''
+/(
+  venv
+  | migrations/*
+)/
+'''

.github/configurations/python_linters/.flake8

@@ -0,0 +1,19 @@
+[flake8]
+max-line-length = 160
+ignore =
+    E121,
+    E125,
+    E203,
+    W504,
+    W503,
+    F401,
+    F403,
+    F405,
+    F841,
+    DJ08,
+    E501,
+    E402,
+
+exclude =
+    */migrations/*,
+    Dockerfile

.github/configurations/python_linters/.isort.cfg

@@ -0,0 +1,8 @@
+[settings]
+multi_line_output = 3
+line_length = 160
+use_parentheses = True
+extend_skip = migrations
+include_trailing_comma = True
+force_grid_wrap = 0
+ensure_newline_before_comments = True

.github/workflows/python-app.yml

@@ -10,28 +10,65 @@ on:
     branches: [ master ]
 
 jobs:
-  build:
+  detect-changes:
+    name: Detect changes
+    runs-on: ubuntu-22.04
+    outputs:
+      python_code: ${{steps.diff_check.outputs.python_code}}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.base_ref }}
+      - uses: actions/checkout@v3
+        with:
+          clean: false
+      - name: Generate diffs
+        id: diff_check
+        run: |
+          git branch -a --list | cat
+          PYTHON_CODE_CHANGES=$(git diff --compact-summary origin/${{ github.base_ref }} -- routingfilter/* | wc -l)
+          echo "::set-output name=python_code::$PYTHON_CODE_CHANGES"
 
-    runs-on: ubuntu-latest
+  build:
+    runs-on: ubuntu-22.04
+    needs: [ "detect-changes" ]
+    if: ${{ needs.detect-changes.outputs.python_code > 0 }}
 
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
         python-version: "3.10"
+    - name: "Cache venv"
+      id: cache_venv
+      uses: actions/cache@v3
+      with:
+        path: venv
+        key: pip-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('requirements.txt') }}
     - name: Install dependencies
+      if: steps.cache_venv.outputs.cache-hit != 'true'
       run: |
-        python -m pip install --upgrade pip
-        pip install flake8 pytest
+        if [ -d "venv" ]; then rm -rf venv; fi
+        python3 -m venv venv
+        source venv/bin/activate
+        pip install --upgrade pip
+        pip install flake8 black isort pytest
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: "Lint with black"
+      run: |
+        source venv/bin/activate
+        black ./routingfilter --config .github/configurations/python_linters/.black --check --diff
     - name: Lint with flake8
       run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+        source venv/bin/activate
+        flake8 ./routingfilter --config .github/configurations/python_linters/.flake8 --show-source
+    - name: "Isort check"
+      run: |
+        source venv/bin/activate
+        isort ./buffalogs --sp .github/configurations/python_linters/.isort.cfg --profile black --filter-files --check-only --diff
     - name: Test with pytest
       run: |
+        source ../venv/bin/activate
         pytest routing_test.py
       working-directory: ./routingfilter

README.md

@@ -22,5 +22,9 @@ See the [online documentation](https://routingfilter.readthedocs.io/en/latest/)
 ### Benchmark tests
 In order to launch the benchmark tests, run ```python routing_benchmark.py```
 
+### Development
+* Install `pip install -r requirements.txt` and `pip install -r requirements_dev.txt` in your local virtual environment
+* Setup pre-commit: `pre-commit install -c .github/.pre-commit-config.yaml`
+
 ### License
-This project is licensed under the **GNU LGPLv3** license.
+This project is licensed under the **GNU LGPLv3** license.

requirements.txt

@@ -1,3 +1,2 @@
-setuptools~=67.7.2
 IPy~=1.1
 macaddress~=2.0.2

requirements_dev.txt

@@ -0,0 +1,5 @@
+black==23.9.1
+flake8==6.1.0
+isort==5.12.0
+pre-commit==3.4.0
+setuptools~=67.7.2

routingfilter/configfilter.py

@@ -1,17 +1,16 @@
-import re
 import logging
+import re
 
-from routingfilter.dictquery import DictQuery
-from IPy import IP
 import macaddress
+from IPy import IP
+from routingfilter.dictquery import DictQuery
 
 
 class ConfigFilter:
-
     def __init__(self, filt):
-        self.type = str(filt.get('type', '')).upper()
-        key = filt.get('key', [])
-        value = filt.get('value', [])
+        self.type = str(filt.get("type", "")).upper()
+        key = filt.get("key", [])
+        value = filt.get("value", [])
         self.key = [key] if isinstance(key, str) else key
         self.value = [value] if isinstance(value, str) or isinstance(value, int) or isinstance(value, float) else value
         self.logger = logging.getLogger(self.__class__.__name__)
@@ -26,7 +25,7 @@ def is_matching(self, data):
         """
         try:
             self.logger.debug(f"Applying filter {self.type} to event: {data}")
-            return getattr(self, '_filter_{}'.format(self.type))(data)
+            return getattr(self, "_filter_{}".format(self.type))(data)
         except AttributeError:
             self.logger.error(f"Invalid filter specified in rules: {self.type}")
             raise
@@ -51,7 +50,7 @@ def _filter_NOT_EQUALS(self, data):
 
     def _filter_EQUALS(self, data):
         for key in self.key:
-            target = DictQuery(data).get(key, '')
+            target = DictQuery(data).get(key, "")
             if isinstance(target, list):
                 for t in target:
                     if self.__check_equals(t):
@@ -70,7 +69,7 @@ def __check_equals(self, target):
 
     def _filter_STARTSWITH(self, data):
         for key in self.key:
-            target = DictQuery(data).get(key, '')
+            target = DictQuery(data).get(key, "")
             if isinstance(target, list):
                 for t in target:
                     if self.__check_startswith(t):
@@ -89,7 +88,7 @@ def __check_startswith(self, target):
 
     def _filter_ENDSWITH(self, data):
         for key in self.key:
-            target = DictQuery(data).get(key, '')
+            target = DictQuery(data).get(key, "")
             if isinstance(target, list):
                 for t in target:
                     if self.__check_endswith(t):
@@ -108,7 +107,7 @@ def __check_endswith(self, target):
 
     def _filter_KEYWORD(self, data):
         for key in self.key:
-            target = DictQuery(data).get(key, '')
+            target = DictQuery(data).get(key, "")
             if isinstance(target, list):
                 for t in target:
                     if self.__check_keyword(t):
@@ -127,7 +126,7 @@ def __check_keyword(self, target):
 
     def _filter_REGEXP(self, data):
         for key in self.key:
-            target = DictQuery(data).get(key, '')
+            target = DictQuery(data).get(key, "")
             if isinstance(target, list):
                 for t in target:
                     if self.__check_regexp(t):
@@ -147,7 +146,7 @@ def _filter_NETWORK(self, data):
         if not data:
             return False
         for key in self.key:
-            target = DictQuery(data).get(key, '0.0.0.0')
+            target = DictQuery(data).get(key, "0.0.0.0")
             if isinstance(target, list):
                 for t in target:
                     if self.__check_network(t):
@@ -173,7 +172,7 @@ def _filter_NOT_NETWORK(self, data):
 
     def _filter_DOMAIN(self, data):
         for key in self.key:
-            target = DictQuery(data).get(key, '')
+            target = DictQuery(data).get(key, "")
             if isinstance(target, list):
                 for t in target:
                     if self.__check_domain(t):
@@ -187,7 +186,7 @@ def __check_domain(self, target):
         target = str(target).lower()
         for value in self.value:
             value = str(value).lower()
-            if target == value or target.endswith('.' + value):
+            if target == value or target.endswith("." + value):
                 return True
         return False
 
@@ -206,7 +205,7 @@ def _filter_LESS_EQ(self, data):
     def __number_comparator(self, data, comparator):
         # Wrapper for filters GREATER, LESS, GREATER_EQ, LESS_EQ
         for key in self.key:
-            target = DictQuery(data).get(key, '')
+            target = DictQuery(data).get(key, "")
             if not target:
                 return False
             if isinstance(target, list):
@@ -250,12 +249,12 @@ def __check_less_eq(self, target):
 
     def _filter_TYPEOF(self, data):
         for key in self.key:
-            target = DictQuery(data).get(key, '')
+            target = DictQuery(data).get(key, "")
             for value in self.value:
                 if self.__check_typeof(target, value):
                     return True
         return False
-    
+
     def __check_typeof(self, target, value):
         if value == "str":
             return type(target) is str
@@ -272,19 +271,19 @@ def __check_typeof(self, target, value):
         elif value == "mac":
             return self.__check_mac_address(target)
         return False
-    
+
     def __check_ip_address(self, target):
-        try: 
+        try:
             if type(target) is int or int(target):
                 return False
         except ValueError:
             # int(value) raises an exception if it isn't int, so can proceed with the ip check
             try:
                 IP(target)
                 return True
-            except:
+            except Exception:
                 return False
-    
+
     def __check_mac_address(self, target):
         if type(target) is int:
             return False
@@ -293,4 +292,3 @@ def __check_mac_address(self, target):
             return True
         except ValueError:
             return False
-

routingfilter/dictquery.py

@@ -1,5 +1,4 @@
 class DictQuery(dict):
-
     # https://www.haykranen.nl/2016/02/13/handling-complex-nested-dicts-in-python/
 
     def get(self, path, default=None):

routingfilter/routing.py

@@ -1,14 +1,13 @@
 import copy
-from datetime import datetime
 import json
 import logging
+from datetime import datetime
+from typing import List, Optional
 
 from routingfilter.configfilter import ConfigFilter
-from typing import List, Optional
 
 
 class Routing:
-
     def __init__(self):
         self.rules = None
         self.variables = {}
@@ -54,7 +53,7 @@ def match(self, event: dict, type_: str = "streams", tag_field_name: str = "tags
         if not isinstance(tags, list):
             tags = [tags]
         tags = set(tags)
-        msg_tags = (tags & streams_tags)
+        msg_tags = tags & streams_tags
         matching_rules = []
 
         # if in routing stream there is an "all" tag I'm checking it for every msg
@@ -216,4 +215,4 @@ def rule_in_routing_history(self, type_, event, rule):
         for key in rule[type_].keys():
             if key in event["certego"]["routing_history"]:
                 return True
-        return False
+        return False