From ff6c5f1153d1e5694ab605b51b7dcfbf6dfaa5dd Mon Sep 17 00:00:00 2001 From: eugenioseveri Date: Tue, 26 Sep 2023 16:05:13 +0200 Subject: [PATCH 1/6] Upgrade CI with linters Added flake8, black, isort, pre-commit Added requirements_dev.txt file --- .github/.pre-commit-config.yaml | 19 ++++++++ .github/configurations/python_linters/.black | 8 ++++ .github/configurations/python_linters/.flake8 | 19 ++++++++ .../configurations/python_linters/.isort.cfg | 8 ++++ .github/workflows/python-app.yml | 45 ++++++++++++++++--- README.md | 6 ++- requirements_dev.txt | 4 ++ 7 files changed, 101 insertions(+), 8 deletions(-) create mode 100644 .github/.pre-commit-config.yaml create mode 100644 .github/configurations/python_linters/.black create mode 100644 .github/configurations/python_linters/.flake8 create mode 100644 .github/configurations/python_linters/.isort.cfg create mode 100644 requirements_dev.txt diff --git a/.github/.pre-commit-config.yaml b/.github/.pre-commit-config.yaml new file mode 100644 index 0000000..bcce425 --- /dev/null +++ b/.github/.pre-commit-config.yaml @@ -0,0 +1,19 @@ +repos: +- repo: https://github.com/pycqa/flake8 + rev: 6.1.0 + hooks: + - id: flake8 + exclude: docs/conf.py + args: ["--config", ".github/configurations/python_linters/.flake8"] + +- repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: + - id: isort + args: ["--settings-path", ".github/configurations/python_linters/.isort.cfg", "--filter-files", "--skip", "venv"] + +- repo: https://github.com/psf/black + rev: 23.9.1 + hooks: + - id: black + args: ["--config", ".github/configurations/python_linters/.black"] diff --git a/.github/configurations/python_linters/.black b/.github/configurations/python_linters/.black new file mode 100644 index 0000000..3319046 --- /dev/null +++ b/.github/configurations/python_linters/.black @@ -0,0 +1,8 @@ +[tool.black] +line-length=160 +extend-exclude=''' +/( + venv + | migrations/* +)/ +''' \ No newline at end of file diff --git a/.github/configurations/python_linters/.flake8 b/.github/configurations/python_linters/.flake8 new file mode 100644 index 0000000..4fe5917 --- /dev/null +++ b/.github/configurations/python_linters/.flake8 @@ -0,0 +1,19 @@ +[flake8] +max-line-length = 160 +ignore = + E121, + E125, + E203, + W504, + W503, + F401, + F403, + F405, + F841, + DJ08, + E501, + E402, + +exclude = + */migrations/*, + Dockerfile \ No newline at end of file diff --git a/.github/configurations/python_linters/.isort.cfg b/.github/configurations/python_linters/.isort.cfg new file mode 100644 index 0000000..562ff47 --- /dev/null +++ b/.github/configurations/python_linters/.isort.cfg @@ -0,0 +1,8 @@ +[settings] +multi_line_output = 3 +line_length = 160 +use_parentheses = True +extend_skip = migrations +include_trailing_comma = True +force_grid_wrap = 0 +ensure_newline_before_comments = True \ No newline at end of file diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 8f7a453..8c5fd8e 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -10,9 +10,29 @@ on: branches: [ master ] jobs: - build: + detect-changes: + name: Detect changes + runs-on: ubuntu-22.04 + outputs: + python_code: ${{steps.diff_check.outputs.python_code}} + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.base_ref }} + - uses: actions/checkout@v3 + with: + clean: false + - name: Generate diffs + id: diff_check + run: | + git branch -a --list | cat + PYTHON_CODE_CHANGES=$(git diff --compact-summary origin/${{ github.base_ref }} -- routingfilter/* | wc -l) + echo "::set-output name=python_code::$PYTHON_CODE_CHANGES" - runs-on: ubuntu-latest + build: + runs-on: ubuntu-22.04 + needs: [ "detect-changes" ] + if: ${{ needs.detect-changes.outputs.python_code > 0 }} steps: - uses: actions/checkout@v3 @@ -20,17 +40,28 @@ jobs: uses: actions/setup-python@v4 with: python-version: "3.10" + - name: "Cache venv" + id: cache_venv + uses: actions/cache@v3 + with: + path: venv + key: pip-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('requirements.txt') }} - name: Install dependencies + if: steps.cache_venv.outputs.cache-hit != 'true' run: | python -m pip install --upgrade pip - pip install flake8 pytest + pip install flake8 black isort pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: "Lint with black" + run: | + source venv/bin/activate + black ./routingfilter --config .github/configurations/python_linters/.black --check --diff - name: Lint with flake8 run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + flake8 ./routingfilter --config .github/configurations/python_linters/.flake8 --show-source + - name: "Isort check" + run: | + isort ./buffalogs --sp .github/configurations/.isort.cfg --profile black --filter-files --check-only --diff - name: Test with pytest run: | pytest routing_test.py diff --git a/README.md b/README.md index 1dea075..c366d35 100644 --- a/README.md +++ b/README.md @@ -22,5 +22,9 @@ See the [online documentation](https://routingfilter.readthedocs.io/en/latest/) ### Benchmark tests In order to launch the benchmark tests, run ```python routing_benchmark.py``` +### Development +* Install `pip install -r requirements.txt` and `pip install -r requirements_dev.txt` in your local virtual environment +* Setup pre-commit: `pre-commit install -c .github/.pre-commit-config.yaml` + ### License -This project is licensed under the **GNU LGPLv3** license. +This project is licensed under the **GNU LGPLv3** license. \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 0000000..1cc57dd --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,4 @@ +black==23.9.1 +flake8==6.1.0 +isort==5.12.0 +pre-commit==3.4.0 \ No newline at end of file From bcdcc10c16b14de2bfa805c4fadbbf17d2c35be3 Mon Sep 17 00:00:00 2001 From: eugenioseveri Date: Tue, 26 Sep 2023 16:08:57 +0200 Subject: [PATCH 2/6] Applied flake8, black, isort --- .github/workflows/python-app.yml | 2 +- routingfilter/configfilter.py | 46 ++++----- routingfilter/dictquery.py | 1 - routingfilter/routing.py | 9 +- routingfilter/routing_benchmark.py | 160 +++++++++++++++-------------- routingfilter/routing_test.py | 109 ++++++++------------ 6 files changed, 151 insertions(+), 176 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 8c5fd8e..00dc37b 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -61,7 +61,7 @@ jobs: flake8 ./routingfilter --config .github/configurations/python_linters/.flake8 --show-source - name: "Isort check" run: | - isort ./buffalogs --sp .github/configurations/.isort.cfg --profile black --filter-files --check-only --diff + isort ./buffalogs --sp .github/configurations/python_linters/.isort.cfg --profile black --filter-files --check-only --diff - name: Test with pytest run: | pytest routing_test.py diff --git a/routingfilter/configfilter.py b/routingfilter/configfilter.py index 1edffb2..43bbe45 100644 --- a/routingfilter/configfilter.py +++ b/routingfilter/configfilter.py @@ -1,17 +1,16 @@ -import re import logging +import re -from routingfilter.dictquery import DictQuery -from IPy import IP import macaddress +from IPy import IP +from routingfilter.dictquery import DictQuery class ConfigFilter: - def __init__(self, filt): - self.type = str(filt.get('type', '')).upper() - key = filt.get('key', []) - value = filt.get('value', []) + self.type = str(filt.get("type", "")).upper() + key = filt.get("key", []) + value = filt.get("value", []) self.key = [key] if isinstance(key, str) else key self.value = [value] if isinstance(value, str) or isinstance(value, int) or isinstance(value, float) else value self.logger = logging.getLogger(self.__class__.__name__) @@ -26,7 +25,7 @@ def is_matching(self, data): """ try: self.logger.debug(f"Applying filter {self.type} to event: {data}") - return getattr(self, '_filter_{}'.format(self.type))(data) + return getattr(self, "_filter_{}".format(self.type))(data) except AttributeError: self.logger.error(f"Invalid filter specified in rules: {self.type}") raise @@ -51,7 +50,7 @@ def _filter_NOT_EQUALS(self, data): def _filter_EQUALS(self, data): for key in self.key: - target = DictQuery(data).get(key, '') + target = DictQuery(data).get(key, "") if isinstance(target, list): for t in target: if self.__check_equals(t): @@ -70,7 +69,7 @@ def __check_equals(self, target): def _filter_STARTSWITH(self, data): for key in self.key: - target = DictQuery(data).get(key, '') + target = DictQuery(data).get(key, "") if isinstance(target, list): for t in target: if self.__check_startswith(t): @@ -89,7 +88,7 @@ def __check_startswith(self, target): def _filter_ENDSWITH(self, data): for key in self.key: - target = DictQuery(data).get(key, '') + target = DictQuery(data).get(key, "") if isinstance(target, list): for t in target: if self.__check_endswith(t): @@ -108,7 +107,7 @@ def __check_endswith(self, target): def _filter_KEYWORD(self, data): for key in self.key: - target = DictQuery(data).get(key, '') + target = DictQuery(data).get(key, "") if isinstance(target, list): for t in target: if self.__check_keyword(t): @@ -127,7 +126,7 @@ def __check_keyword(self, target): def _filter_REGEXP(self, data): for key in self.key: - target = DictQuery(data).get(key, '') + target = DictQuery(data).get(key, "") if isinstance(target, list): for t in target: if self.__check_regexp(t): @@ -147,7 +146,7 @@ def _filter_NETWORK(self, data): if not data: return False for key in self.key: - target = DictQuery(data).get(key, '0.0.0.0') + target = DictQuery(data).get(key, "0.0.0.0") if isinstance(target, list): for t in target: if self.__check_network(t): @@ -173,7 +172,7 @@ def _filter_NOT_NETWORK(self, data): def _filter_DOMAIN(self, data): for key in self.key: - target = DictQuery(data).get(key, '') + target = DictQuery(data).get(key, "") if isinstance(target, list): for t in target: if self.__check_domain(t): @@ -187,7 +186,7 @@ def __check_domain(self, target): target = str(target).lower() for value in self.value: value = str(value).lower() - if target == value or target.endswith('.' + value): + if target == value or target.endswith("." + value): return True return False @@ -206,7 +205,7 @@ def _filter_LESS_EQ(self, data): def __number_comparator(self, data, comparator): # Wrapper for filters GREATER, LESS, GREATER_EQ, LESS_EQ for key in self.key: - target = DictQuery(data).get(key, '') + target = DictQuery(data).get(key, "") if not target: return False if isinstance(target, list): @@ -250,12 +249,12 @@ def __check_less_eq(self, target): def _filter_TYPEOF(self, data): for key in self.key: - target = DictQuery(data).get(key, '') + target = DictQuery(data).get(key, "") for value in self.value: if self.__check_typeof(target, value): return True return False - + def __check_typeof(self, target, value): if value == "str": return type(target) is str @@ -272,9 +271,9 @@ def __check_typeof(self, target, value): elif value == "mac": return self.__check_mac_address(target) return False - + def __check_ip_address(self, target): - try: + try: if type(target) is int or int(target): return False except ValueError: @@ -282,9 +281,9 @@ def __check_ip_address(self, target): try: IP(target) return True - except: + except Exception: return False - + def __check_mac_address(self, target): if type(target) is int: return False @@ -293,4 +292,3 @@ def __check_mac_address(self, target): return True except ValueError: return False - diff --git a/routingfilter/dictquery.py b/routingfilter/dictquery.py index f286adf..e92590a 100644 --- a/routingfilter/dictquery.py +++ b/routingfilter/dictquery.py @@ -1,5 +1,4 @@ class DictQuery(dict): - # https://www.haykranen.nl/2016/02/13/handling-complex-nested-dicts-in-python/ def get(self, path, default=None): diff --git a/routingfilter/routing.py b/routingfilter/routing.py index 2ec5b3f..df1b5db 100644 --- a/routingfilter/routing.py +++ b/routingfilter/routing.py @@ -1,14 +1,13 @@ import copy -from datetime import datetime import json import logging +from datetime import datetime +from typing import List, Optional from routingfilter.configfilter import ConfigFilter -from typing import List, Optional class Routing: - def __init__(self): self.rules = None self.variables = {} @@ -54,7 +53,7 @@ def match(self, event: dict, type_: str = "streams", tag_field_name: str = "tags if not isinstance(tags, list): tags = [tags] tags = set(tags) - msg_tags = (tags & streams_tags) + msg_tags = tags & streams_tags matching_rules = [] # if in routing stream there is an "all" tag I'm checking it for every msg @@ -216,4 +215,4 @@ def rule_in_routing_history(self, type_, event, rule): for key in rule[type_].keys(): if key in event["certego"]["routing_history"]: return True - return False \ No newline at end of file + return False diff --git a/routingfilter/routing_benchmark.py b/routingfilter/routing_benchmark.py index 569983b..a97a2bd 100644 --- a/routingfilter/routing_benchmark.py +++ b/routingfilter/routing_benchmark.py @@ -1,28 +1,31 @@ -from datetime import datetime import json import os +from datetime import datetime from typing import List from routingfilter.routing import Routing + def load_test_data(name): """Load a JSON test file from 'test_data' folder, given its name (extension excluded), and parse it into a dictionary.""" - with open(os.path.join('test_data', name + '.json')) as file: + with open(os.path.join("test_data", name + ".json")) as file: data = json.load(file) return data + MAX_RULE = 1000 MAX_EVENT = 100 MAX_LIST_VALUES = 100 MAX_LIST_VALUES_EVENT = 10 -class RoutingBenchMark(): - """Class to test the routing performance in order to monitor the trend of the execution time after new features or changes""" - + +class RoutingBenchMark: + """Class to test the routing performance in order to monitor the trend of the execution time after new features or changes""" + def test1_EQUALS_no_key_match(self): """Performance test, for the EQUALS routing filter type, with: - - 100 same rules (type: EQUALS) - - 100 messages with 50 fields different from 'wheel_model' + - 100 same rules (type: EQUALS) + - 100 messages with 50 fields different from 'wheel_model' """ routing = Routing() my_dict = load_test_data("benchmark_rule_equals_dict") @@ -41,8 +44,8 @@ def test1_EQUALS_no_key_match(self): def test2_EQUALS_key_exists(self): """Performance test, for the EQUALS routing filter type, with: - - 100 same rules (type: EQUALS) - - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' + - 100 same rules (type: EQUALS) + - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' """ routing = Routing() my_dict = load_test_data("benchmark_rule_equals_dict") @@ -60,12 +63,12 @@ def test2_EQUALS_key_exists(self): routing.match(benchmark_event_1) end_time = datetime.now() print(f"{self.test2_EQUALS_key_exists.__name__}: {(end_time - start_time).total_seconds()}") - + def test3_EQUALS_list_values(self): """Performance test, for the EQUALS routing filter type, with: - - 100 same rules (type: EQUALS) - - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' - - 100 different values in the rules (no matches with the message field) + - 100 same rules (type: EQUALS) + - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' + - 100 different values in the rules (no matches with the message field) """ routing = Routing() my_dict = load_test_data("benchmark_rule_equals_dict") @@ -88,7 +91,7 @@ def test3_EQUALS_list_values(self): def test4_EQUALS_values_message(self): """Performance test, for the EQUALS routing filter type, with: - - 100 values in the "wheel_model" of the rule and of the message, but none of them triggers a match (no 'Superlight') + - 100 values in the "wheel_model" of the rule and of the message, but none of them triggers a match (no 'Superlight') """ routing = Routing() my_dict = load_test_data("benchmark_rule_equals_dict") @@ -104,7 +107,7 @@ def test4_EQUALS_values_message(self): benchmark_event_1 = load_test_data("benchmark_event_1") benchmark_event_1.update({"wheel_model": []}) for i in range(MAX_LIST_VALUES_EVENT): - benchmark_event_1["wheel_model"].append("no_match-" + str(MAX_LIST_VALUES+i)) + benchmark_event_1["wheel_model"].append("no_match-" + str(MAX_LIST_VALUES + i)) start_time = datetime.now() # Sending 100 messages to the routing for i in range(MAX_EVENT): @@ -114,12 +117,12 @@ def test4_EQUALS_values_message(self): def test1_STARTSWITH_no_key_match(self): """Performance test, for the STARTSWITH routing filter type, with: - - 100 same rules (type: STARTSWITH) - - 100 messages with 50 fields different from 'wheel_model' + - 100 same rules (type: STARTSWITH) + - 100 messages with 50 fields different from 'wheel_model' """ routing = Routing() my_dict = load_test_data("benchmark_rule_startswith_dict") - # Create 100 EQUALS same rules + # Create 100 EQUALS same rules rule = load_test_data("benchmark_rule_startswith") for i in range(MAX_RULE): rule["streams"]["rules"]["mountain_bike"].append(my_dict) @@ -134,8 +137,8 @@ def test1_STARTSWITH_no_key_match(self): def test2_STARTSWITH_key_exists(self): """Performance test, for the STARTSWITH routing filter type, with: - - 100 same rules (type: STARTSWITH) - - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' + - 100 same rules (type: STARTSWITH) + - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' """ routing = Routing() my_dict = load_test_data("benchmark_rule_startswith_dict") @@ -156,9 +159,9 @@ def test2_STARTSWITH_key_exists(self): def test3_STARTSWITH_list_values(self): """Performance test, for the STARTSWITH routing filter type, with: - - 100 same rules (type: STARTSWITH) - - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' - - 100 different values in the rules (no matches with the message field) + - 100 same rules (type: STARTSWITH) + - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' + - 100 different values in the rules (no matches with the message field) """ routing = Routing() my_dict = load_test_data("benchmark_rule_startswith_dict") @@ -181,7 +184,7 @@ def test3_STARTSWITH_list_values(self): def test4_STARTSWITH_values_message(self): """Performance test, for the EQUALS routing filter type, with: - - 100 values in the "wheel_model" of the rule and of the message, but none of them triggers a match (no 'Superlight') + - 100 values in the "wheel_model" of the rule and of the message, but none of them triggers a match (no 'Superlight') """ routing = Routing() my_dict = load_test_data("benchmark_rule_startswith_dict") @@ -207,12 +210,12 @@ def test4_STARTSWITH_values_message(self): def test1_ENDSWITH_no_key_match(self): """Performance test, for the ENDSWITH routing filter type, with: - - 100 same rules (type: ENDSWITH) - - 100 messages with 50 fields different from 'wheel_model' + - 100 same rules (type: ENDSWITH) + - 100 messages with 50 fields different from 'wheel_model' """ routing = Routing() my_dict = load_test_data("benchmark_rule_endswith_dict") - # Create 100 EQUALS same rules + # Create 100 EQUALS same rules rule = load_test_data("benchmark_rule_endswith") for i in range(MAX_RULE): rule["streams"]["rules"]["mountain_bike"].append(my_dict) @@ -227,8 +230,8 @@ def test1_ENDSWITH_no_key_match(self): def test2_ENDSWITH_key_exists(self): """Performance test, for the ENDSWITH routing filter type, with: - - 100 same rules (type: ENDSWITH) - - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' + - 100 same rules (type: ENDSWITH) + - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' """ routing = Routing() my_dict = load_test_data("benchmark_rule_endswith_dict") @@ -249,9 +252,9 @@ def test2_ENDSWITH_key_exists(self): def test3_ENDSWITH_list_values(self): """Performance test, for the ENDSWITH routing filter type, with: - - 100 same rules (type: ENDSWITH) - - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' - - 100 different values in the rules (no matches with the message field) + - 100 same rules (type: ENDSWITH) + - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' + - 100 different values in the rules (no matches with the message field) """ routing = Routing() my_dict = load_test_data("benchmark_rule_endswith_dict") @@ -274,7 +277,7 @@ def test3_ENDSWITH_list_values(self): def test4_ENDSWITH_values_message(self): """Performance test, for the ENDSWITH routing filter type, with: - - 100 values in the "wheel_model" of the rule and of the message, but none of them triggers a match (no 'Superlight') + - 100 values in the "wheel_model" of the rule and of the message, but none of them triggers a match (no 'Superlight') """ routing = Routing() my_dict = load_test_data("benchmark_rule_endswith_dict") @@ -300,12 +303,12 @@ def test4_ENDSWITH_values_message(self): def test1_KEYWORD_no_key_match(self): """Performance test, for the KEYWORD routing filter type, with: - - 100 same rules (type: KEYWORD) - - 100 messages with 50 fields different from 'wheel_model' + - 100 same rules (type: KEYWORD) + - 100 messages with 50 fields different from 'wheel_model' """ routing = Routing() my_dict = load_test_data("benchmark_rule_keyword_dict") - # Create 100 EQUALS same rules + # Create 100 EQUALS same rules rule = load_test_data("benchmark_rule_keyword") for i in range(MAX_RULE): rule["streams"]["rules"]["mountain_bike"].append(my_dict) @@ -320,8 +323,8 @@ def test1_KEYWORD_no_key_match(self): def test2_KEYWORD_key_exists(self): """Performance test, for the KEYWORD routing filter type, with: - - 100 same rules (type: KEYWORD) - - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' + - 100 same rules (type: KEYWORD) + - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' """ routing = Routing() my_dict = load_test_data("benchmark_rule_keyword_dict") @@ -342,9 +345,9 @@ def test2_KEYWORD_key_exists(self): def test3_KEYWORD_list_values(self): """Performance test, for the KEYWORD routing filter type, with: - - 100 same rules (type: KEYWORD) - - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' - - 100 different values in the rules (no matches with the message field) + - 100 same rules (type: KEYWORD) + - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' + - 100 different values in the rules (no matches with the message field) """ routing = Routing() my_dict = load_test_data("benchmark_rule_keyword_dict") @@ -367,7 +370,7 @@ def test3_KEYWORD_list_values(self): def test4_KEYWORD_values_message(self): """Performance test, for the KEYWORD routing filter type, with: - - 100 values in the "wheel_model" of the rule and of the message, but none of them triggers a match (no 'Superlight') + - 100 values in the "wheel_model" of the rule and of the message, but none of them triggers a match (no 'Superlight') """ routing = Routing() my_dict = load_test_data("benchmark_rule_keyword_dict") @@ -393,12 +396,12 @@ def test4_KEYWORD_values_message(self): def test1_REGEXP_no_key_match(self): """Performance test, for the REGEXP routing filter type, with: - - 100 same rules (type: REGEXP) - - 100 messages with 50 fields different from 'wheel_model' + - 100 same rules (type: REGEXP) + - 100 messages with 50 fields different from 'wheel_model' """ routing = Routing() my_dict = load_test_data("benchmark_rule_regexp_dict") - # Create 100 EQUALS same rules + # Create 100 EQUALS same rules rule = load_test_data("benchmark_rule_regexp") for i in range(MAX_RULE): rule["streams"]["rules"]["mountain_bike"].append(my_dict) @@ -413,8 +416,8 @@ def test1_REGEXP_no_key_match(self): def test2_REGEXP_key_exists(self): """Performance test, for the REGEXP routing filter type, with: - - 100 same rules (type: REGEXP) - - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' + - 100 same rules (type: REGEXP) + - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' """ routing = Routing() my_dict = load_test_data("benchmark_rule_regexp_dict") @@ -435,9 +438,9 @@ def test2_REGEXP_key_exists(self): def test3_REGEXP_list_values(self): """Performance test, for the REGEXP routing filter type, with: - - 100 same rules (type: REGEXP) - - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' - - 100 different values in the rules (no matches with the message field) + - 100 same rules (type: REGEXP) + - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' + - 100 different values in the rules (no matches with the message field) """ routing = Routing() my_dict = load_test_data("benchmark_rule_regexp_dict") @@ -460,7 +463,7 @@ def test3_REGEXP_list_values(self): def test4_REGEXP_values_message(self): """Performance test, for the REGEXP routing filter type, with: - - 100 values in the "wheel_model" of the rule and of the message, but none of them triggers a match (no 'Superlight') + - 100 values in the "wheel_model" of the rule and of the message, but none of them triggers a match (no 'Superlight') """ routing = Routing() my_dict = load_test_data("benchmark_rule_regexp_dict") @@ -486,12 +489,12 @@ def test4_REGEXP_values_message(self): def test1_NETWORK_no_key_match(self): """Performance test, for the NETWORK routing filter type, with: - - 100 same rules (type: NETWORK) - - 100 messages with 50 fields different from 'wheel_model' + - 100 same rules (type: NETWORK) + - 100 messages with 50 fields different from 'wheel_model' """ routing = Routing() my_dict = load_test_data("benchmark_rule_network_dict") - # Create 100 EQUALS same rules + # Create 100 EQUALS same rules rule = load_test_data("benchmark_rule_network") for i in range(MAX_RULE): rule["streams"]["rules"]["mountain_bike"].append(my_dict) @@ -506,8 +509,8 @@ def test1_NETWORK_no_key_match(self): def test2_NETWORK_key_exists(self): """Performance test, for the NETWORK routing filter type, with: - - 100 same rules (type: NETWORK) - - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' + - 100 same rules (type: NETWORK) + - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' """ routing = Routing() my_dict = load_test_data("benchmark_rule_network_dict") @@ -528,9 +531,9 @@ def test2_NETWORK_key_exists(self): def test3_NETWORK_list_values(self): """Performance test, for the NETWORK routing filter type, with: - - 100 same rules (type: NETWORK) - - 100 messages with 50 fields (one of them 'wheel_model' but not in the 10.10.10.0/24 network - - 100 different values in the rules (no matches with the message field) + - 100 same rules (type: NETWORK) + - 100 messages with 50 fields (one of them 'wheel_model' but not in the 10.10.10.0/24 network + - 100 different values in the rules (no matches with the message field) """ routing = Routing() my_dict = load_test_data("benchmark_rule_network_dict") @@ -554,7 +557,7 @@ def test3_NETWORK_list_values(self): def test4_NETWORK_values_message(self): """Performance test, for the REGEXP routing filter type, with: - - 100 values in the "wheel_model" of the rule and of the message, but none of them triggers a match (no network 10.10.10.0/24) + - 100 values in the "wheel_model" of the rule and of the message, but none of them triggers a match (no network 10.10.10.0/24) """ routing = Routing() my_dict = load_test_data("benchmark_rule_network_dict") @@ -581,12 +584,12 @@ def test4_NETWORK_values_message(self): def test1_DOMAIN_no_key_match(self): """Performance test, for the DOMAIN routing filter type, with: - - 100 same rules (type: DOMAIN) - - 100 messages with 50 fields different from 'wheel_model' + - 100 same rules (type: DOMAIN) + - 100 messages with 50 fields different from 'wheel_model' """ routing = Routing() my_dict = load_test_data("benchmark_rule_domain_dict") - # Create 100 EQUALS same rules + # Create 100 EQUALS same rules rule = load_test_data("benchmark_rule_domain") for i in range(MAX_RULE): rule["streams"]["rules"]["mountain_bike"].append(my_dict) @@ -601,8 +604,8 @@ def test1_DOMAIN_no_key_match(self): def test2_DOMAIN_key_exists(self): """Performance test, for the DOMAIN routing filter type, with: - - 100 same rules (type: DOMAIN) - - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' + - 100 same rules (type: DOMAIN) + - 100 messages with 50 fields (one of them 'wheel_model' but not 'Superlight' """ routing = Routing() my_dict = load_test_data("benchmark_rule_domain_dict") @@ -623,9 +626,9 @@ def test2_DOMAIN_key_exists(self): def test3_DOMAIN_list_values(self): """Performance test, for the DOMAIN routing filter type, with: - - 100 same rules (type: DOMAIN) - - 100 messages with 50 fields (one of them 'wheel_model' but not in the google.com domain - - 100 different values in the rules (no matches with the message field) + - 100 same rules (type: DOMAIN) + - 100 messages with 50 fields (one of them 'wheel_model' but not in the google.com domain + - 100 different values in the rules (no matches with the message field) """ routing = Routing() my_dict = load_test_data("benchmark_rule_domain_dict") @@ -648,7 +651,7 @@ def test3_DOMAIN_list_values(self): def test4_DOMAIN_values_message(self): """Performance test, for the DOMAIN routing filter type, with: - - 100 values in the "wheel_model" of the rule and of the message, but none of them triggers a match (no domain google.com) + - 100 values in the "wheel_model" of the rule and of the message, but none of them triggers a match (no domain google.com) """ routing = Routing() my_dict = load_test_data("benchmark_rule_domain_dict") @@ -674,12 +677,12 @@ def test4_DOMAIN_values_message(self): def test1_GREATER_no_key_match(self): """Performance test, for the GREATER routing filter type, with: - - 100 same rules (type: GREATER) - - 100 messages with 50 fields different from 'wheel_model' + - 100 same rules (type: GREATER) + - 100 messages with 50 fields different from 'wheel_model' """ routing = Routing() my_dict = load_test_data("benchmark_rule_greater_dict") - # Create 100 EQUALS same rules + # Create 100 EQUALS same rules rule = load_test_data("benchmark_rule_greater") for i in range(MAX_RULE): rule["streams"]["rules"]["mountain_bike"].append(my_dict) @@ -694,8 +697,8 @@ def test1_GREATER_no_key_match(self): def test2_GREATER_key_exists(self): """Performance test, for the GREATER routing filter type, with: - - 100 same rules (type: GREATER) - - 100 messages with 50 fields (one of them 'wheel_model' but not greater than 1) + - 100 same rules (type: GREATER) + - 100 messages with 50 fields (one of them 'wheel_model' but not greater than 1) """ routing = Routing() my_dict = load_test_data("benchmark_rule_greater_dict") @@ -716,9 +719,9 @@ def test2_GREATER_key_exists(self): def test3_GREATER_list_values(self): """Performance test, for the GREATER routing filter type, with: - - 100 same rules (type: GREATER) - - 100 messages with 50 fields (one of them 'wheel_model' but not greater than 1 - - 100 different values in the rules (no matches with the message field) + - 100 same rules (type: GREATER) + - 100 messages with 50 fields (one of them 'wheel_model' but not greater than 1 + - 100 different values in the rules (no matches with the message field) """ routing = Routing() my_dict = load_test_data("benchmark_rule_greater_dict") @@ -741,7 +744,7 @@ def test3_GREATER_list_values(self): def test4_GREATER_values_message(self): """Performance test, for the GREATER routing filter type, with: - - 100 values in the "wheel_model" of the rule and of the message, but none of them triggers a match (no domain google.com) + - 100 values in the "wheel_model" of the rule and of the message, but none of them triggers a match (no domain google.com) """ routing = Routing() my_dict = load_test_data("benchmark_rule_greater_dict") @@ -801,5 +804,6 @@ def main(): routing_benchmark.test3_GREATER_list_values() routing_benchmark.test4_GREATER_values_message() + if __name__ == "__main__": main() diff --git a/routingfilter/routing_test.py b/routingfilter/routing_test.py index 010d447..d7af8a9 100644 --- a/routingfilter/routing_test.py +++ b/routingfilter/routing_test.py @@ -1,6 +1,6 @@ import copy -import os import json +import os import unittest from routingfilter.routing import Routing @@ -8,7 +8,7 @@ def load_test_data(name): """Load a JSON test file from 'test_data' folder, given its name (extension excluded), and parse it into a dictionary.""" - with open(os.path.join('test_data', name + '.json')) as file: + with open(os.path.join("test_data", name + ".json")) as file: data = json.load(file) return data @@ -63,15 +63,15 @@ def test_multiple_rule_loading(self): load_test_data("test_rule_5_exists"), ] self.routing.load_from_dicts(rule_list_duplicate) - self.assertEqual(len(self.routing.get_rules()['streams']['rules']['mountain_bike']), 3) - self.assertEqual(len(self.routing.get_rules()['streams']['rules']['ip_traffic']), 2) + self.assertEqual(len(self.routing.get_rules()["streams"]["rules"]["mountain_bike"]), 3) + self.assertEqual(len(self.routing.get_rules()["streams"]["rules"]["ip_traffic"]), 2) # Verify that the only first rule matches rule_all = load_test_data("test_rule_2_all_equals") filter_2 = copy.deepcopy(rule_all["streams"]["rules"]["all"][0]) filter_2["streams"] = {"Other": None} rule_all["streams"]["rules"]["all"].append(filter_2) self.routing.load_from_dicts([rule_all]) - self.assertDictEqual(self.routing.match(self.test_event_1)[0]["output"], {'Workshop': {'workers_needed': 1}}) + self.assertDictEqual(self.routing.match(self.test_event_1)[0]["output"], {"Workshop": {"workers_needed": 1}}) def test_match_streams_none(self): self.routing.load_from_dicts([load_test_data("test_rule_25_routing_history_streams_none")]) @@ -98,7 +98,7 @@ def test_routing_history(self): self.routing.match(self.test_event_1) self.assertTrue("Workshop" in self.test_event_1["certego"]["routing_history"]) self.assertTrue("TyreFit" in self.test_event_1["certego"]["routing_history"]) - + def test_routing_history_stream_none(self): self.routing.load_from_dicts([load_test_data("test_rule_1_equals")]) self.routing.match(self.test_event_1) @@ -126,7 +126,7 @@ def test_routing_history_same_rule_twice(self): def test_double_stream(self): self.routing.load_from_dicts([load_test_data("test_rule_28_double_stream")]) res = self.routing.match(self.test_event_1) - self.assertDictEqual({'Workshop': {'workers_needed': 1}, 'Lab': {'workers_needed': 2}}, res[0]["output"]) + self.assertDictEqual({"Workshop": {"workers_needed": 1}, "Lab": {"workers_needed": 2}}, res[0]["output"]) self.assertTrue("Workshop" in self.test_event_1["certego"]["routing_history"]) self.assertTrue("Lab" in self.test_event_1["certego"]["routing_history"]) self.assertEqual(2, len(res[0]["output"])) @@ -175,7 +175,7 @@ def test_special_tag_all(self): self.assertDictEqual(self.routing.match(self.test_event_1)[0], load_test_data("test_event_1_rule_1_response")) self.assertDictEqual(self.routing.match(self.test_event_2)[0], load_test_data("test_event_1_rule_1_response")) self.assertEqual(self.routing.match(self.test_event_3), []) - + def test_special_tag_all2(self): test_rule_1_equals = load_test_data("test_rule_1_equals") test_rule_2_all_equals = load_test_data("test_rule_2_all_equals") @@ -213,7 +213,7 @@ def test_single_filters_EQUALS(self): self.routing.load_from_dicts([load_test_data("test_rule_1_equals")]) # EQUALS self.assertTrue(self.routing.match(self.test_event_1)) self.assertFalse(self.routing.match(self.test_event_3)) - + def test_single_filters_NOT_EQUALS(self): self.routing.load_from_dicts([load_test_data("test_rule_1_not_equals")]) # NOT EQUALS self.assertFalse(self.routing.match(self.test_event_1)) @@ -249,7 +249,7 @@ def test_single_filters_GREATER(self): event_2["tags"] = "mountain_bike" event_2["price"] = "600a" self.routing.match(event_2) - + def test_single_filters_LESS(self): self.routing.load_from_dicts([load_test_data("test_rule_13_less")]) # LESS self.assertFalse(self.routing.match(self.test_event_1)) @@ -266,7 +266,7 @@ def test_single_filter_EXIST(self): self.assertTrue(self.routing.match(self.test_event_1)) self.assertFalse(self.routing.match(self.test_event_3)) self.assertTrue(self.routing.match(self.test_event_10)) - + def test_single_filter_NOT_EXISTS(self): self.routing.load_from_dicts([load_test_data("test_rule_6_not_exists")]) # NOT_EXISTS self.assertFalse(self.routing.match(self.test_event_1)) @@ -296,49 +296,49 @@ def test_event_with_lists_as_fields(self): def test_single_filter_TYPEOF_exception(self): # if self.value is a list, it returns True if almost one type is correct - self.routing.load_from_dicts([load_test_data("test_rule_15_typeof_exception")]) # "value": ["str", "int", "dict"] - self.assertTrue(self.routing.match(self.test_event_8)) # value: "str" - self.assertTrue(self.routing.match(self.test_event_11)) # value: "int" - self.assertTrue(self.routing.match(self.test_event_13)) # value: "dict" + self.routing.load_from_dicts([load_test_data("test_rule_15_typeof_exception")]) # "value": ["str", "int", "dict"] + self.assertTrue(self.routing.match(self.test_event_8)) # value: "str" + self.assertTrue(self.routing.match(self.test_event_11)) # value: "int" + self.assertTrue(self.routing.match(self.test_event_13)) # value: "dict" self.assertFalse(self.routing.match(self.test_event_12)) def test_single_filter_TYPEOF_str(self): - self.routing.load_from_dicts([load_test_data("test_rule_16_typeof_str")]) # is_str + self.routing.load_from_dicts([load_test_data("test_rule_16_typeof_str")]) # is_str self.assertTrue(self.routing.match(self.test_event_8)) - self.assertFalse(self.routing.match(self.test_event_10)) # is_not_str + self.assertFalse(self.routing.match(self.test_event_10)) # is_not_str def test_single_filter_TYPEOF_int(self): - self.routing.load_from_dicts([load_test_data("test_rule_17_typeof_int")]) # is_int + self.routing.load_from_dicts([load_test_data("test_rule_17_typeof_int")]) # is_int self.assertTrue(self.routing.match(self.test_event_11)) - self.assertFalse(self.routing.match(self.test_event_8)) # is_not_int + self.assertFalse(self.routing.match(self.test_event_8)) # is_not_int def test_single_filter_TYPEOF_bool(self): - self.routing.load_from_dicts([load_test_data("test_rule_18_typeof_bool")]) # is_bool + self.routing.load_from_dicts([load_test_data("test_rule_18_typeof_bool")]) # is_bool self.assertTrue(self.routing.match(self.test_event_12)) - self.assertFalse(self.routing.match(self.test_event_8)) # is_not_bool - + self.assertFalse(self.routing.match(self.test_event_8)) # is_not_bool + def test_single_filter_TYPEOF_list(self): self.routing.load_from_dicts([load_test_data("test_rule_19_typeof_list")]) # is_list self.assertTrue(self.routing.match(self.test_event_10)) - self.assertFalse(self.routing.match(self.test_event_8)) # is_not_list + self.assertFalse(self.routing.match(self.test_event_8)) # is_not_list def test_single_filter_TYPEOF_dict(self): self.routing.load_from_dicts([load_test_data("test_rule_20_typeof_dict")]) # is_dict self.assertTrue(self.routing.match(self.test_event_13)) - self.assertFalse(self.routing.match(self.test_event_11)) # is_not_dict + self.assertFalse(self.routing.match(self.test_event_11)) # is_not_dict def test_single_filter_TYPEOF_ip_address(self): self.routing.load_from_dicts([load_test_data("test_rule_21_typeof_ip")]) # is_ipv4 self.assertTrue(self.routing.match(self.test_event_15)) - self.assertTrue(self.routing.match(self.test_event_16)) # is_ipv6 - self.assertFalse(self.routing.match(self.test_event_11)) # insert an integer - self.assertFalse(self.routing.match(self.test_event_14)) # insert a string with a number: ex. "8" - self.assertFalse(self.routing.match(self.test_event_12)) # is_not_ip + self.assertTrue(self.routing.match(self.test_event_16)) # is_ipv6 + self.assertFalse(self.routing.match(self.test_event_11)) # insert an integer + self.assertFalse(self.routing.match(self.test_event_14)) # insert a string with a number: ex. "8" + self.assertFalse(self.routing.match(self.test_event_12)) # is_not_ip def test_single_filter_TYPEOF_mac_address(self): - self.routing.load_from_dicts([load_test_data("test_rule_22_typeof_mac")]) # is_mac + self.routing.load_from_dicts([load_test_data("test_rule_22_typeof_mac")]) # is_mac self.assertTrue(self.routing.match(self.test_event_17)) - self.assertFalse(self.routing.match(self.test_event_16)) # is_not_mac + self.assertFalse(self.routing.match(self.test_event_16)) # is_not_mac # key doesn't exist self.assertFalse(self.routing.match(self.test_event_5)) @@ -347,53 +347,28 @@ def test_variables_no_list(self): self.assertFalse(self.routing.match(self.test_event_4)) self.routing.load_from_dicts([load_test_data("test_rule_23_network_variables")], variables={"$INTERNAL_IPS": "192.168.1.0/24"}) self.assertTrue(self.routing.match(self.test_event_4)) - - def test_variables_list_one_element(self): + + def test_variables_list_one_element(self): self.routing.load_from_dicts([load_test_data("test_rule_26_network_variables_list1")], variables={"$INTERNAL_IPS": "192.168.1.0/24"}) self.assertTrue(self.routing.match(self.test_event_4)) - + def test_variables_list_more_elements(self): - self.routing.load_from_dicts([load_test_data("test_rule_27_network_variables_list2")], variables={"$INTERNAL_IPS": ["192.168.1.0/24", "192.168.2.0/24"]}) + self.routing.load_from_dicts( + [load_test_data("test_rule_27_network_variables_list2")], variables={"$INTERNAL_IPS": ["192.168.1.0/24", "192.168.2.0/24"]} + ) self.assertTrue(self.routing.match(self.test_event_4)) def test_rule_in_routing_history(self): event = {"certego": {"routing_history": {}}} rule = { - "filters": [ - { - "type": "EQUALS", - "key": "wheel_model", - "description": "Carbon fiber wheels needs manual truing", - "value": [ - "Superlight", - "RacePro" - ] - } - ], - "streams": { - } - } + "filters": [{"type": "EQUALS", "key": "wheel_model", "description": "Carbon fiber wheels needs manual truing", "value": ["Superlight", "RacePro"]}], + "streams": {}, + } self.assertFalse(self.routing.rule_in_routing_history("streams", event, rule)) event = {"certego": {"routing_history": {"Workshop": "2023-06-06T18:00:00.000Z"}}} self.assertFalse(self.routing.rule_in_routing_history("streams", event, rule)) rule = { - "filters": [ - { - "type": "EQUALS", - "key": "wheel_model", - "description": "Carbon fiber wheels needs manual truing", - "value": [ - "Superlight", - "RacePro" - ] - } - ], - "streams": { - "Workshop": { - "workers_needed": 1 - } - } - } + "filters": [{"type": "EQUALS", "key": "wheel_model", "description": "Carbon fiber wheels needs manual truing", "value": ["Superlight", "RacePro"]}], + "streams": {"Workshop": {"workers_needed": 1}}, + } self.assertTrue(self.routing.rule_in_routing_history("streams", event, rule)) - - From be43bdf3cad98224bd663c3492496d08c4a4cef9 Mon Sep 17 00:00:00 2001 From: eugenioseveri Date: Tue, 26 Sep 2023 16:15:04 +0200 Subject: [PATCH 3/6] Trying to fix venv in CI --- .github/workflows/python-app.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 00dc37b..ac04be8 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -49,7 +49,10 @@ jobs: - name: Install dependencies if: steps.cache_venv.outputs.cache-hit != 'true' run: | - python -m pip install --upgrade pip + if [ -d "venv" ]; then rm -rf venv; fi + python3 -m venv venv + source venv/bin/activate + pip install --upgrade pip pip install flake8 black isort pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: "Lint with black" @@ -58,9 +61,11 @@ jobs: black ./routingfilter --config .github/configurations/python_linters/.black --check --diff - name: Lint with flake8 run: | + source venv/bin/activate flake8 ./routingfilter --config .github/configurations/python_linters/.flake8 --show-source - name: "Isort check" run: | + source venv/bin/activate isort ./buffalogs --sp .github/configurations/python_linters/.isort.cfg --profile black --filter-files --check-only --diff - name: Test with pytest run: | From ade7849523bc127600603a9421c7bd127da39a65 Mon Sep 17 00:00:00 2001 From: eugenioseveri Date: Tue, 26 Sep 2023 16:16:57 +0200 Subject: [PATCH 4/6] Trying to fix venv in CI --- .github/workflows/python-app.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index ac04be8..36eeaab 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -69,5 +69,6 @@ jobs: isort ./buffalogs --sp .github/configurations/python_linters/.isort.cfg --profile black --filter-files --check-only --diff - name: Test with pytest run: | + source venv/bin/activate pytest routing_test.py working-directory: ./routingfilter From 7dddf47caa5c29116f26b2fc9f04009919864dc2 Mon Sep 17 00:00:00 2001 From: eugenioseveri Date: Tue, 26 Sep 2023 16:19:01 +0200 Subject: [PATCH 5/6] Trying to fix venv in CI --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 36eeaab..f1be82a 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -69,6 +69,6 @@ jobs: isort ./buffalogs --sp .github/configurations/python_linters/.isort.cfg --profile black --filter-files --check-only --diff - name: Test with pytest run: | - source venv/bin/activate + source ../venv/bin/activate pytest routing_test.py working-directory: ./routingfilter From 3beeb4027e3bc65366aee186c5249ecff75a2bfe Mon Sep 17 00:00:00 2001 From: eugenioseveri Date: Wed, 27 Sep 2023 10:05:09 +0200 Subject: [PATCH 6/6] Moved setuptools to requirements_dev --- requirements.txt | 1 - requirements_dev.txt | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index c39b584..4194d8d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ -setuptools~=67.7.2 IPy~=1.1 macaddress~=2.0.2 \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt index 1cc57dd..dff2a6c 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1,5 @@ black==23.9.1 flake8==6.1.0 isort==5.12.0 -pre-commit==3.4.0 \ No newline at end of file +pre-commit==3.4.0 +setuptools~=67.7.2 \ No newline at end of file