Merge pull request #14 from subugoe/extend-statuses

Extend statuses

.env

@@ -1,17 +1,34 @@
-OPERANDI_DB_NAME=operandi_db
-OPERANDI_DB_ROOT_USER=db_operandi
-OPERANDI_DB_ROOT_PASS=db_operandi
-OPERANDI_DB_URL=mongodb://db_operandi:db_operandi@localhost:27017
-OPERANDI_HARVESTER_DEFAULT_USERNAME=harvester_operandi
-OPERANDI_HARVESTER_DEFAULT_PASSWORD=harvester_operandi
-OPERANDI_HPC_USERNAME=mmustaf
-OPERANDI_HPC_SSH_KEYPATH=/home/mm/.ssh/gwdg-cluster
-OPERANDI_HPC_PROJECT_NAME=operandi
+DOCKER_OPERANDI_NETWORK_NAME=operandi_network
+DOCKER_OPERANDI_NETWORK_MTU=1450
+
 OPERANDI_LOGS_DIR=/tmp/operandi_logs
-OPERANDI_RABBITMQ_CONFIG_JSON=./src/rabbitmq_definitions.json
-OPERANDI_RABBITMQ_URL=amqp://operandi_user:operandi_password@localhost:5672/
 OPERANDI_SERVER_BASE_DIR=/tmp/operandi_data
-OPERANDI_SERVER_DEFAULT_USERNAME=server_operandi
-OPERANDI_SERVER_DEFAULT_PASSWORD=server_operandi
-OPERANDI_SERVER_URL_LIVE=http://localhost:8000
-OPERANDI_SERVER_URL_LOCAL=http://0.0.0.0:8000
+
+OPERANDI_HARVESTER_DEFAULT_USERNAME=operandi
+OPERANDI_HARVESTER_DEFAULT_PASSWORD=operandi
+
+OPERANDI_HPC_USERNAME=mmustaf
+OPERANDI_HPC_PROJECT_NAME=operandi
+OPERANDI_HPC_SSH_KEYPATH=/home/root/.ssh/gwdg_hpc_key
+
+OPERANDI_SERVER_DEFAULT_USERNAME=operandi
+OPERANDI_SERVER_DEFAULT_PASSWORD=operandi
+OPERANDI_SERVER_HOST=operandi-server-docker-host
+OPERANDI_SERVER_PORT=8000
+OPERANDI_SERVER_URL=http://${OPERANDI_SERVER_HOST}.${DOCKER_OPERANDI_NETWORK_NAME}:${OPERANDI_SERVER_PORT}
+
+MONGODB_NAME=operandi_db
+MONGODB_USER=operandi
+MONGODB_PASS=operandi
+MONGODB_HOST=operandi-mongodb-docker-host
+MONGODB_PORT=27017
+MONGODB_URL=mongodb://${MONGODB_USER}:${MONGODB_PASS}@${MONGODB_HOST}.${DOCKER_OPERANDI_NETWORK_NAME}:${MONGODB_PORT}
+
+RABBITMQ_FEATURE_FLAGS=quorum_queue,implicit_default_bindings,classic_mirrored_queue_version
+RABBITMQ_USER=operandi
+RABBITMQ_PASS=operandi
+RABBITMQ_HOST=operandi-rabbitmq-docker-host
+RABBITMQ_PORT=5672
+RABBITMQ_PORT_UI=15672
+RABBITMQ_PORT_CLI=25672
+RABBITMQ_URL=amqp://${RABBITMQ_USER}:${RABBITMQ_PASS}@${RABBITMQ_HOST}.${DOCKER_OPERANDI_NETWORK_NAME}:${RABBITMQ_PORT}

.github/workflows/ci_cd.yml

@@ -70,7 +70,7 @@ jobs:
     name: Run operandi tests
     needs: build-native
     env:
-      ENV_FILE: ./.github/workflows/tests/.env
+      ENV_FILE: ./.github/workflows/test.env
     strategy:
       fail-fast: false
       matrix:
@@ -89,15 +89,8 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install operandi dependencies
-        run: |
-          sudo apt-get update && sudo apt-get -y install make
-          python3 -m pip install --upgrade pip setuptools  
-          pip3 install -U pip wheel
-          pip install -r tests/requirements.txt
-
-      - name: Install operandi modules
-        run: make install
+      - name: Install make
+        run: sudo apt-get update && sudo apt-get -y install make
 
       - name: Import variables from env file
         shell: bash
@@ -114,39 +107,11 @@ jobs:
           echo "$SSH_KEY_HPC" > /home/runner/.ssh/key_hpc
           chmod 600 /home/runner/.ssh/key_hpc
 
-      - name: start RabbitMQ Server
-        run: docker compose -f ./docker-compose.yml --env-file ${{ env.ENV_FILE }} up -d operandi-rabbitmq
-
-      - name: start MongoDB
-        run: docker compose -f ./docker-compose.yml --env-file ${{ env.ENV_FILE }} up -d operandi-mongodb
-
-      - name: wait starting of RabbitMQ and MongoDB
-        run: sleep 45
-
-      - name: run utils tests
-        run: |
-          export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
-          pytest tests/tests_utils/test_*.py
-
-      - name: run broker tests
-        run: |
-          export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
-          pytest tests/tests_broker/test_*.py
-
-      - name: run server tests
-        run: |
-          export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
-          pytest tests/tests_server/test_*.py
-
-      - name: run harvester tests
-        run: |
-          export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
-          pytest tests/tests_harvester/test_*.py
-
       - name: run integration tests
         run: |
-          export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
-          pytest tests/integration_tests/test_*.py -s -v
+          make start-services-test
+          make run-integration-tests
+          make stop-services-test
 
   build-and-push-docker-images:
     name: Push ${{matrix.services.module}} to image registry
@@ -157,8 +122,8 @@ jobs:
       fail-fast: true
       matrix:
         services: [
-          { module: "broker", dockerfile: "Dockerfile_broker" },
-          { module: "server", dockerfile: "Dockerfile_server" }
+          { module: "broker", dockerfile: "Dockerfile" },
+          { module: "server", dockerfile: "Dockerfile" }
         ]
         os: [ ubuntu-latest ]
     runs-on: ${{ matrix.os }}
@@ -190,8 +155,8 @@ jobs:
       - name: Build and push Docker image
         uses: docker/build-push-action@v4
         with:
-          context: ./src
-          file: ./src/${{matrix.services.dockerfile}}
+          context: .
+          file: ./${{matrix.services.dockerfile}}
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }} 

.github/workflows/test.env

@@ -0,0 +1,34 @@
+DOCKER_OPERANDI_NETWORK_NAME=operandi_network
+DOCKER_OPERANDI_NETWORK_MTU=1450
+
+OPERANDI_LOGS_DIR=/tmp/operandi_logs_test
+OPERANDI_SERVER_BASE_DIR=/tmp/operandi_data_test
+
+OPERANDI_HARVESTER_DEFAULT_USERNAME=operandi_test
+OPERANDI_HARVESTER_DEFAULT_PASSWORD=operandi_test
+
+OPERANDI_HPC_USERNAME=mmustaf
+OPERANDI_HPC_PROJECT_NAME=operandi_tests_cicd
+OPERANDI_HPC_SSH_KEYPATH=/home/runner/.ssh/key_hpc
+
+OPERANDI_SERVER_DEFAULT_USERNAME=operandi_test
+OPERANDI_SERVER_DEFAULT_PASSWORD=operandi_test
+OPERANDI_SERVER_HOST=operandi-server-docker-host-test
+OPERANDI_SERVER_PORT=8000
+OPERANDI_SERVER_URL=http://${OPERANDI_SERVER_HOST}.${DOCKER_OPERANDI_NETWORK_NAME}:${OPERANDI_SERVER_PORT}
+
+MONGODB_NAME=operandi_db_test
+MONGODB_USER=operandi_test
+MONGODB_PASS=operandi_test
+MONGODB_HOST=operandi-mongodb-docker-host-test
+MONGODB_PORT=27017
+MONGODB_URL=mongodb://${MONGODB_USER}:${MONGODB_PASS}@${MONGODB_HOST}.${DOCKER_OPERANDI_NETWORK_NAME}:${MONGODB_PORT}
+
+RABBITMQ_FEATURE_FLAGS=quorum_queue,implicit_default_bindings,classic_mirrored_queue_version
+RABBITMQ_USER=operandi_test
+RABBITMQ_PASS=operandi_test
+RABBITMQ_HOST=operandi-rabbitmq-docker-host-test
+RABBITMQ_PORT=5672
+RABBITMQ_PORT_UI=15672
+RABBITMQ_PORT_CLI=25672
+RABBITMQ_URL=amqp://${RABBITMQ_USER}:${RABBITMQ_PASS}@${RABBITMQ_HOST}.${DOCKER_OPERANDI_NETWORK_NAME}:${RABBITMQ_PORT}

Dockerfile

@@ -0,0 +1,56 @@
+FROM ubuntu:22.04 as operandi_base
+
+MAINTAINER OPERANDI
+ENV DEBIAN_FRONTEND noninteractive
+ENV PYTHONIOENCODING utf8
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
+WORKDIR /build-operandi
+
+COPY src ./src
+COPY Makefile .
+
+RUN apt-get update && apt-get install -y \
+    curl \
+    git  \
+    make \
+    python3 \
+    python3-dev \
+    python3-pip \
+    python3-venv \
+    software-properties-common \
+    sudo \
+    time \
+    wget
+
+RUN python3 -m pip install --upgrade pip setuptools
+RUN pip3 install -U pip wheel
+
+RUN python3 -m pip install -r /build-operandi/src/utils/requirements.txt --ignore-installed
+RUN pip3 install /build-operandi/src/utils
+
+RUN python3 -m pip install -r /build-operandi/src/broker/requirements.txt --ignore-installed
+RUN pip3 install /build-operandi/src/broker
+
+RUN python3 -m pip install -r /build-operandi/src/harvester/requirements.txt --ignore-installed
+RUN pip3 install /build-operandi/src/harvester
+
+RUN python3 -m pip install -r /build-operandi/src/server/requirements.txt --ignore-installed
+RUN pip3 install /build-operandi/src/server
+
+RUN mkdir /operandi-data && chmod 777 /operandi-data
+RUN mkdir /operandi-logs && chmod 777 /operandi-logs
+
+RUN echo "Operandi build success"
+RUN operandi-broker --version
+RUN operandi-server --version
+RUN operandi-harvester --version
+
+FROM operandi_base as operandi_tests
+WORKDIR /build-operandi
+COPY Makefile .
+COPY tests ./tests
+RUN python3 -m pip install -r ./tests/requirements.txt
+
+CMD ["yes"]

Makefile

@@ -6,13 +6,13 @@ PIP3_INSTALL = pip3 install
 BUILD_ORDER = src/utils src/server src/broker src/harvester
 UNINSTALL_ORDER = operandi_harvester operandi_broker operandi_server operandi_utils
 
-ifneq (,$(wildcard ./.env))
-    include ./.env
-endif
+# ifneq (,$(wildcard ./.env))
+#    include ./.env
+# endif
 
-ifneq (,$(wildcard ./tests/.env))
-    include ./tests/.env
-endif
+# ifneq (,$(wildcard ./tests/.env))
+#    include ./tests/.env
+# endif
 
 help:
 	@echo ""
@@ -111,27 +111,42 @@ start-harvester-dummy:
 	export $(shell sed 's/=.*//' ./.env)
 	operandi-harvester start-dummy --address http://localhost
 
-run-tests: run-tests-utils run-tests-broker run-tests-server run-tests-harvester run-tests-integration
+DOCKER_COMPOSE = docker compose
+start-services-test:
+	$(DOCKER_COMPOSE) --file docker-compose.yml --env-file tests/.env up -d
+stop-services-test:
+	$(DOCKER_COMPOSE) --file docker-compose.yml --env-file tests/.env down --remove-orphans
 
-run-tests-utils:
-	export $(shell sed 's/=.*//' ./tests/.env)
-	pytest tests/tests_utils/test_*.py -v
+INTEGRATION_TEST_IN_DOCKER = docker exec operandi_tests_container
+run-tests-db:
+	$(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_integration_1_db' -v
+
+run-tests-rabbitmq:
+	$(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_integration_2_rabbitmq' -v
+
+run-tests-hpc:
+	$(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_integration_3_hpc' -v
 
 run-tests-broker:
-	export $(shell sed 's/=.*//' ./tests/.env)
-	pytest tests/tests_broker/test_*.py -v
+	$(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_integration_4_broker' -v
+
+run-tests-server:
+	$(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_integration_5_server' -v
 
 run-tests-harvester:
-	export $(shell sed 's/=.*//' ./tests/.env)
-	pytest tests/tests_harvester/test_*.py -v
+	$(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_integration_6_harvester' -v
 
-run-tests-server:
-	export $(shell sed 's/=.*//' ./tests/.env)
-	pytest tests/tests_server/test_*.py -v
+run-integration-tests: run-tests-db run-tests-rabbitmq run-tests-hpc run-tests-broker run-tests-server run-tests-harvester
+
+run-all-tests:
+	start-services-test
+	run-integration-tests
+	stop-services-test:
 
-run-tests-integration:
-	export $(shell sed 's/=.*//' ./tests/.env)
-	pytest tests/integration_tests/test_*.py -s -v
+run-all-tests-cicd:
+	start-services-test
+	-run-integration-tests
+	stop-services-test
 
 pyclean:
 	rm -f **/*.pyc