From cc05107f6d8e60faeb3a85a2d0bf3acb713209d9 Mon Sep 17 00:00:00 2001 From: lvalics Date: Wed, 7 Feb 2024 11:23:19 +0200 Subject: [PATCH 1/7] Create example.env --- dj_backend_server/example.env | 75 +++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 dj_backend_server/example.env diff --git a/dj_backend_server/example.env b/dj_backend_server/example.env new file mode 100644 index 00000000..2161dd7b --- /dev/null +++ b/dj_backend_server/example.env @@ -0,0 +1,75 @@ +########################################################## + +# Edit values for your site. +# your app secret key +SECRET_KEY='ADD-YOUR-CUSTOM-KEY-HERE' +# For openai +OPENAI_API_KEY=YOURKEY +# add IP what you allow like superadmin +ALLOWED_HOSTS=localhost,0.0.0.0 +# Use * only in dev environment +#ALLOWED_HOSTS=* +# Your SITE URL +APP_URL='https://YOUR-URL-HERE' + +########################################################## + +# "azure" | "openai" | llama2 +OPENAI_API_TYPE=openai +OPENAI_API_MODEL=gpt-4-1106-preview +OPENAI_API_TEMPERATURE=1 + +# If using azure +# AZURE_OPENAI_API_BASE= +# AZURE_OPENAI_API_KEY= +# AZURE_OPENAI_API_VERSION=2023-03-15-preview +# AZURE_OPENAI_EMBEDDING_MODEL_NAME= +# AZURE_OPENAI_DEPLOYMENT_NAME= +# AZURE_OPENAI_COMPLETION_MODEL=gpt-35-turbo + +# "azure" | "openai" | llama2 +EMBEDDING_PROVIDER=openai + +# Vector Store, PINECONE|QDRANT +STORE=QDRANT + + +# if using pinecone +# PINECONE_API_KEY= +# PINECONE_ENV= +# VECTOR_STORE_INDEX_NAME= + + +# if using qdrant +QDRANT_URL=http://qdrant:6333 + + +# optional, defaults to 15 +MAX_PAGES_CRAWL=150 + +# --- these will change if you decide to start testing the software +CELERY_BROKER_URL=redis://redis:6379/ +CELERY_RESULT_BACKEND=redis://redis:6379/ +DATABASE_NAME=openchat +DATABASE_USER=dbuser +DATABASE_PASSWORD=dbpass +DATABASE_HOST=mysql +DATABASE_PORT=3306 + +# use 'external' if you want to use below services. +PDF_LIBRARY = 'external' + +#PDF API - OCRWebService.com (REST API). https://www.ocrwebservice.com/api/restguide +#Extract text from scanned images and PDF documents and convert into editable formats. +#Please create new account with ocrwebservice.com via http://www.ocrwebservice.com/account/signup and get license code +OCR_LICCODE = 'LICENSE-CODE' +OCR_USERNAME = 'USERNAME' +OCR_LANGUAGE = 'english' +# Advantage to clean up the OCR text which can be messy and full with garbage, but will generate a cost with LLM if is paid. Use carefully. +# Use 1 to enable, 0 to disable. +OCR_LLM = '1' + +# retrieval_qa | conversation_retrieval, retrieval_qa works better with azure openai +# if you want to use the conversation_retrieval | retrieval_qa chain +CHAIN_TYPE=conversation_retrieval + From d0a8f0e64fc9f7bcf94dd3a23aa81ef29be3b070 Mon Sep 17 00:00:00 2001 From: lvalics Date: Tue, 13 Feb 2024 11:51:44 +0000 Subject: [PATCH 2/7] Small typos and compatibility checks. --- dj_backend_server/Makefile | 27 +++++++++++++++------------ dj_backend_server/docker-compose.yaml | 1 - dj_backend_server/requirements.txt | 5 ++--- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/dj_backend_server/Makefile b/dj_backend_server/Makefile index 93c3dcfd..22baf405 100644 --- a/dj_backend_server/Makefile +++ b/dj_backend_server/Makefile @@ -1,6 +1,7 @@ # Makefile to Run docker-compose for Django App -# first run "make preinstall" to create .env file and venv environment +# first run "make pre-install" to create .env file and venv environment +# Be sure you are in dj_backend_serrver directory to run the commands # you will need to edit .env file, it will be created from example.env # after .env is done, run "make install" to create run the nginx and docker-compose. # doublecheck if the nginx.conf is correct, if not, edit it and run "make install" again @@ -34,7 +35,7 @@ endif case $$yn in \ [Yy]*) cp example.env .env; \ echo "The .env file has been created from example.env."; \ - echo "Please edit the .env file and set the values for OPENAI_API_KEY and APP_URL.";; \ + echo "Please edit the .env file and set the values for OPENAI_API_KEY, SECRET_KEY and APP_URL. ALLOWED_HOSTS is not mandatory.";; \ *) echo "Exiting. Please create a .env file manually."; exit 1;; \ esac; \ else \ @@ -44,6 +45,8 @@ endif fi venv: + @echo "Checking for Python venv module..." + @python3 -c "import venv" 2>/dev/null || (echo "\033[1;31mPython venv module is not installed. Please install it using your package manager.\033[0m" && exit 1) @if [ ! -d "venv" ]; then \ echo "\033[1;32mCreating a virtual environment...\033[0m"; \ python3 -m venv venv; \ @@ -71,7 +74,7 @@ nginx: fi down: - $(DOCKER_COMPOSE) down --remove-orphans + "$(DOCKER_COMPOSE)" down --remove-orphans ifeq ($(OS), Darwin) # macOS OPEN_COMMAND := open @@ -90,37 +93,37 @@ else endif install: nginx - $(DOCKER_COMPOSE) up -d + "$(DOCKER_COMPOSE)" up -d pre-install: .env venv install-requirements # celery -A dj_backend_server worker --loglevel=info & dev-start: - $(DOCKER_COMPOSE) -f docker-compose.linux.yaml up -d + "$(DOCKER_COMPOSE)" -f docker-compose.linux.yaml up -d @echo "$(shell tput setaf 3)Waiting for 20 seconds before opening the browser...$(shell tput sgr0)" sleep 20 $(OPEN_COMMAND) http://0.0.0.0:8000/ dev-stop: - $(DOCKER_COMPOSE) down --remove-orphans + "$(DOCKER_COMPOSE)" down --remove-orphans kill -9 $$(pgrep -f "celery -A dj_backend_server") kill -9 $$(pgrep -f "python3 manage.py runserver") @echo $$(tput setaf 3)"Services stopped."$$(tput sgr0) force_migrate: @echo $(shell tput setaf 2)"Running migrations inside the Docker container..." - $(DOCKER) exec -it web python manage.py makemigrations web - $(DOCKER) exec -it web python manage.py migrate + "$(DOCKER)" exec -it web python manage.py makemigrations web + "$(DOCKER)" exec -it web python manage.py migrate logs: - $(DOCKER_COMPOSE) logs -f + "$(DOCKER_COMPOSE)" logs -f exec: - $(DOCKER) exec -u 0 -it oc_web /bin/bash + "$(DOCKER)" exec -u 0 -it oc_web /bin/bash restart: - $(DOCKER) restart oc_web - $(DOCKER) restart oc_celery + "$(DOCKER)" restart oc_web + "$(DOCKER)" restart oc_celery make logs .PHONY: install .env venv activate-venv install-requirements down dev-start dev-stop nginx diff --git a/dj_backend_server/docker-compose.yaml b/dj_backend_server/docker-compose.yaml index bf2d58ab..e5120629 100644 --- a/dj_backend_server/docker-compose.yaml +++ b/dj_backend_server/docker-compose.yaml @@ -3,7 +3,6 @@ version: '3.9' services: mysql: restart: unless-stopped - platform: linux/arm64/v8 image: "mysql:8" ports: - "3307:3306" diff --git a/dj_backend_server/requirements.txt b/dj_backend_server/requirements.txt index 4a6e2cef..dc66b862 100644 --- a/dj_backend_server/requirements.txt +++ b/dj_backend_server/requirements.txt @@ -31,8 +31,8 @@ grpcio-tools==1.56.2 h11==0.14.0 h2==4.1.0 hpack==4.0.0 -httpcore1.0.2 -httpx=0.25.2 +httpcore==1.0.2 +httpx==0.25.2 hyperframe==6.0.1 idna==3.6 kombu==5.3.1 @@ -84,4 +84,3 @@ wcwidth==0.2.6 yarl==1.9.2 django-cors-headers==4.3.1 ollama==0.1.4 - From be07395a5813f3ac532fcbbfd4aa6a6535932b11 Mon Sep 17 00:00:00 2001 From: lvalics Date: Tue, 13 Feb 2024 11:52:16 +0000 Subject: [PATCH 3/7] Revert "Small typos and compatibility checks." This reverts commit d0a8f0e64fc9f7bcf94dd3a23aa81ef29be3b070. --- dj_backend_server/Makefile | 27 ++++++++++++--------------- dj_backend_server/docker-compose.yaml | 1 + dj_backend_server/requirements.txt | 5 +++-- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/dj_backend_server/Makefile b/dj_backend_server/Makefile index 22baf405..93c3dcfd 100644 --- a/dj_backend_server/Makefile +++ b/dj_backend_server/Makefile @@ -1,7 +1,6 @@ # Makefile to Run docker-compose for Django App -# first run "make pre-install" to create .env file and venv environment -# Be sure you are in dj_backend_serrver directory to run the commands +# first run "make preinstall" to create .env file and venv environment # you will need to edit .env file, it will be created from example.env # after .env is done, run "make install" to create run the nginx and docker-compose. # doublecheck if the nginx.conf is correct, if not, edit it and run "make install" again @@ -35,7 +34,7 @@ endif case $$yn in \ [Yy]*) cp example.env .env; \ echo "The .env file has been created from example.env."; \ - echo "Please edit the .env file and set the values for OPENAI_API_KEY, SECRET_KEY and APP_URL. ALLOWED_HOSTS is not mandatory.";; \ + echo "Please edit the .env file and set the values for OPENAI_API_KEY and APP_URL.";; \ *) echo "Exiting. Please create a .env file manually."; exit 1;; \ esac; \ else \ @@ -45,8 +44,6 @@ endif fi venv: - @echo "Checking for Python venv module..." - @python3 -c "import venv" 2>/dev/null || (echo "\033[1;31mPython venv module is not installed. Please install it using your package manager.\033[0m" && exit 1) @if [ ! -d "venv" ]; then \ echo "\033[1;32mCreating a virtual environment...\033[0m"; \ python3 -m venv venv; \ @@ -74,7 +71,7 @@ nginx: fi down: - "$(DOCKER_COMPOSE)" down --remove-orphans + $(DOCKER_COMPOSE) down --remove-orphans ifeq ($(OS), Darwin) # macOS OPEN_COMMAND := open @@ -93,37 +90,37 @@ else endif install: nginx - "$(DOCKER_COMPOSE)" up -d + $(DOCKER_COMPOSE) up -d pre-install: .env venv install-requirements # celery -A dj_backend_server worker --loglevel=info & dev-start: - "$(DOCKER_COMPOSE)" -f docker-compose.linux.yaml up -d + $(DOCKER_COMPOSE) -f docker-compose.linux.yaml up -d @echo "$(shell tput setaf 3)Waiting for 20 seconds before opening the browser...$(shell tput sgr0)" sleep 20 $(OPEN_COMMAND) http://0.0.0.0:8000/ dev-stop: - "$(DOCKER_COMPOSE)" down --remove-orphans + $(DOCKER_COMPOSE) down --remove-orphans kill -9 $$(pgrep -f "celery -A dj_backend_server") kill -9 $$(pgrep -f "python3 manage.py runserver") @echo $$(tput setaf 3)"Services stopped."$$(tput sgr0) force_migrate: @echo $(shell tput setaf 2)"Running migrations inside the Docker container..." - "$(DOCKER)" exec -it web python manage.py makemigrations web - "$(DOCKER)" exec -it web python manage.py migrate + $(DOCKER) exec -it web python manage.py makemigrations web + $(DOCKER) exec -it web python manage.py migrate logs: - "$(DOCKER_COMPOSE)" logs -f + $(DOCKER_COMPOSE) logs -f exec: - "$(DOCKER)" exec -u 0 -it oc_web /bin/bash + $(DOCKER) exec -u 0 -it oc_web /bin/bash restart: - "$(DOCKER)" restart oc_web - "$(DOCKER)" restart oc_celery + $(DOCKER) restart oc_web + $(DOCKER) restart oc_celery make logs .PHONY: install .env venv activate-venv install-requirements down dev-start dev-stop nginx diff --git a/dj_backend_server/docker-compose.yaml b/dj_backend_server/docker-compose.yaml index e5120629..bf2d58ab 100644 --- a/dj_backend_server/docker-compose.yaml +++ b/dj_backend_server/docker-compose.yaml @@ -3,6 +3,7 @@ version: '3.9' services: mysql: restart: unless-stopped + platform: linux/arm64/v8 image: "mysql:8" ports: - "3307:3306" diff --git a/dj_backend_server/requirements.txt b/dj_backend_server/requirements.txt index dc66b862..4a6e2cef 100644 --- a/dj_backend_server/requirements.txt +++ b/dj_backend_server/requirements.txt @@ -31,8 +31,8 @@ grpcio-tools==1.56.2 h11==0.14.0 h2==4.1.0 hpack==4.0.0 -httpcore==1.0.2 -httpx==0.25.2 +httpcore1.0.2 +httpx=0.25.2 hyperframe==6.0.1 idna==3.6 kombu==5.3.1 @@ -84,3 +84,4 @@ wcwidth==0.2.6 yarl==1.9.2 django-cors-headers==4.3.1 ollama==0.1.4 + From e4f662591abf2dcddfa3bae6340b22027b320428 Mon Sep 17 00:00:00 2001 From: lvalics Date: Tue, 13 Feb 2024 11:53:21 +0000 Subject: [PATCH 4/7] Small typos --- dj_backend_server/docker-compose.yaml | 1 - dj_backend_server/requirements.txt | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dj_backend_server/docker-compose.yaml b/dj_backend_server/docker-compose.yaml index bf2d58ab..e5120629 100644 --- a/dj_backend_server/docker-compose.yaml +++ b/dj_backend_server/docker-compose.yaml @@ -3,7 +3,6 @@ version: '3.9' services: mysql: restart: unless-stopped - platform: linux/arm64/v8 image: "mysql:8" ports: - "3307:3306" diff --git a/dj_backend_server/requirements.txt b/dj_backend_server/requirements.txt index 4a6e2cef..e035d0e1 100644 --- a/dj_backend_server/requirements.txt +++ b/dj_backend_server/requirements.txt @@ -31,8 +31,8 @@ grpcio-tools==1.56.2 h11==0.14.0 h2==4.1.0 hpack==4.0.0 -httpcore1.0.2 -httpx=0.25.2 +httpcore==1.0.2 +httpx==0.25.2 hyperframe==6.0.1 idna==3.6 kombu==5.3.1 From d55156bebb9ac0761e0aab230adcd03ebbd7a0fa Mon Sep 17 00:00:00 2001 From: lvalics Date: Tue, 13 Feb 2024 19:22:29 +0000 Subject: [PATCH 5/7] mostly small typos, translation and in docker the oc_name --- dj_backend_server/api/data_sources/pdf_handler.py | 6 ++++-- dj_backend_server/docker-compose.yaml | 9 ++++++--- dj_backend_server/requirements.txt | 3 +-- .../onboarding/other-data-sources-website.html | 10 +++------- dj_backend_server/web/templates/onboarding/step-0.html | 3 +-- dj_backend_server/web/templates/onboarding/step-2.html | 2 +- 6 files changed, 16 insertions(+), 17 deletions(-) diff --git a/dj_backend_server/api/data_sources/pdf_handler.py b/dj_backend_server/api/data_sources/pdf_handler.py index 6890fa8d..9bf20312 100644 --- a/dj_backend_server/api/data_sources/pdf_handler.py +++ b/dj_backend_server/api/data_sources/pdf_handler.py @@ -222,7 +222,7 @@ def txt_to_vectordb(shared_folder: str, namespace: str, delete_folder_flag: bool docs = text_splitter.split_documents(raw_docs) - print("external files docs -->", docs); + # print("external files docs -->", docs); if not docs: print("No documents were processed successfully.") @@ -230,7 +230,9 @@ def txt_to_vectordb(shared_folder: str, namespace: str, delete_folder_flag: bool embeddings = get_embeddings() + print(f"Initializing vector store for namespace: {namespace} with {len(docs)} documents.") init_vector_store(docs, embeddings, StoreOptions(namespace=namespace)) + print(f"Vector store initialized successfully for namespace: {namespace}.") print(f'Folder need or not to delete. {delete_folder_flag}') # Delete folder if flag is set @@ -243,7 +245,7 @@ def txt_to_vectordb(shared_folder: str, namespace: str, delete_folder_flag: bool # pdf_data_source.save() failed_job = FailedJob(uuid=str(uuid4()), connection='default', queue='default', payload='txt_to_vectordb', exception=str(e),failed_at=timezone.now()) failed_job.save() - print(e) + print(f"Failed to initialize vector store for namespace: {namespace}. Exception: {e}") traceback.print_exc() diff --git a/dj_backend_server/docker-compose.yaml b/dj_backend_server/docker-compose.yaml index e5120629..88495ffd 100644 --- a/dj_backend_server/docker-compose.yaml +++ b/dj_backend_server/docker-compose.yaml @@ -3,6 +3,7 @@ version: '3.9' services: mysql: restart: unless-stopped + container_name: oc_mysql image: "mysql:8" ports: - "3307:3306" @@ -19,6 +20,7 @@ services: qdrant: image: qdrant/qdrant + container_name: oc_qdrant ports: - 6333:6333 - 6334:6334 @@ -31,7 +33,7 @@ services: build: context: . dockerfile: Dockerfile - container_name: web + container_name: oc_web ports: - "8001:8000" volumes: @@ -52,6 +54,7 @@ services: adminer: image: adminer + container_name: oc_adminer ports: - "8080:8080" environment: @@ -65,7 +68,7 @@ services: build: context: . dockerfile: Dockerfile - container_name: celery + container_name: oc_celery volumes: - ./website_data_sources:/app/website_data_sources # - ./llama-2-7b-chat.ggmlv3.q4_K_M.bin:/app/llama-2-7b-chat.ggmlv3.q4_K_M.bin:ro @@ -79,7 +82,7 @@ services: redis: image: redis:latest - container_name: redis_cache + container_name: oc_redis_cache ports: - "6379:6379" volumes: diff --git a/dj_backend_server/requirements.txt b/dj_backend_server/requirements.txt index e035d0e1..9c686faa 100644 --- a/dj_backend_server/requirements.txt +++ b/dj_backend_server/requirements.txt @@ -19,8 +19,7 @@ click-repl==0.3.0 cryptography==41.0.3 dataclasses-json==0.5.14 Django==4.2.3 -django-rest-swagger -djangorestframework +django-rest-swagger==2.2.0 dnspython==2.4.1 drf-spectacular==0.27.1 drf_spectacular.extensions==0.0.2 diff --git a/dj_backend_server/web/templates/onboarding/other-data-sources-website.html b/dj_backend_server/web/templates/onboarding/other-data-sources-website.html index 4a6c4875..f2d01891 100644 --- a/dj_backend_server/web/templates/onboarding/other-data-sources-website.html +++ b/dj_backend_server/web/templates/onboarding/other-data-sources-website.html @@ -79,20 +79,16 @@

{% trans 'Website information
-
{% trans "Just to make sure we are on - the same page" %}
+
{% trans "Just to make sure we are on the same page" %}
- {% trans "Sometimes, we might face challenges when trying to crawl certain websites, - especially the ones built using JavaScript (Single-Page Applications). However, we're - currently working on adding headless browsing to our system so that we can support all - kinds of websites." %} + {% trans "Sometimes, we might face challenges when trying to crawl certain websites, especially the ones built using JavaScript (Single-Page Applications). However, we're currently working on adding headless browsing to our system so that we can support all kinds of websites." %}
<- {% trans "Back" %} - +
diff --git a/dj_backend_server/web/templates/onboarding/step-0.html b/dj_backend_server/web/templates/onboarding/step-0.html index b3775875..db34b838 100644 --- a/dj_backend_server/web/templates/onboarding/step-0.html +++ b/dj_backend_server/web/templates/onboarding/step-0.html @@ -56,8 +56,7 @@

{% trans "Let's set up your f

{% trans "You provide the system with data" %}

-
{% trans "It could be a website, pdf files, and soon you will have the option to - integrate with many more" %}
+
{% trans "It could be a website, pdf files, and soon you will have the option to integrate with many more" %}
  • diff --git a/dj_backend_server/web/templates/onboarding/step-2.html b/dj_backend_server/web/templates/onboarding/step-2.html index 580998c7..2cf4804e 100644 --- a/dj_backend_server/web/templates/onboarding/step-2.html +++ b/dj_backend_server/web/templates/onboarding/step-2.html @@ -78,7 +78,7 @@

    {% trans "Website information
    -
    {% trans "Just to make sure we are on the same page" %} 🫶 +
    {% trans "Just to make sure we are on the same page" %}
    {% trans "We might not be able to crawl some websites, especially websites that are built using JS (SPA), we are working on adding headless browsing to support all sorts of websites." %} From 8669c4ba188fec2c17187c1c6ad0f67a42300e06 Mon Sep 17 00:00:00 2001 From: lvalics Date: Wed, 14 Feb 2024 18:29:42 +0000 Subject: [PATCH 6/7] We need to be sure that Vector Database exist (QDrant now). The issue is on large websites, not create the VD. --- .../api/utils/init_vector_store.py | 28 +++++++++++++++++++ dj_backend_server/web/workers/crawler.py | 7 ++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/dj_backend_server/api/utils/init_vector_store.py b/dj_backend_server/api/utils/init_vector_store.py index 36458b93..ea7a96c0 100644 --- a/dj_backend_server/api/utils/init_vector_store.py +++ b/dj_backend_server/api/utils/init_vector_store.py @@ -128,3 +128,31 @@ def delete_from_vector_store(namespace: str, filter_criteria: dict) -> None: else: raise NotImplementedError(f"Delete operation is not implemented for the store type: {store_type}") + +def ensure_vector_database_exists(namespace): + store_type = StoreType[os.environ['STORE']] + try: + if store_type == StoreType.QDRANT: + client = QdrantClient(url=os.environ['QDRANT_URL']) + for attempt in range(3): + existing_collections = client.get_collections().collections + if namespace not in existing_collections: + print(f"Namespace '{namespace}' does not exist. Attempting to create.") + vectors_config = models.VectorParams( + size=1536, # Using 1536-dimensional vectors, adjust as necessary + distance=models.Distance.COSINE # Using cosine distance, adjust as necessary + ) + client.create_collection(collection_name=namespace, vectors_config=vectors_config) + # Recheck if the namespace was successfully created + if namespace in client.get_collections().collections: + print(f"Namespace '{namespace}' successfully created.") + return + else: + print(f"Failed to create namespace '{namespace}' on attempt {attempt + 1}.") + else: + print(f"Namespace '{namespace}' exists.") + return + raise Exception(f"Failed to ensure or create namespace '{namespace}' after 3 attempts.") + except Exception as e: + print(f"Failed to ensure vector database exists for namespace {namespace}: {e}") + diff --git a/dj_backend_server/web/workers/crawler.py b/dj_backend_server/web/workers/crawler.py index 35779d91..18e4437b 100644 --- a/dj_backend_server/web/workers/crawler.py +++ b/dj_backend_server/web/workers/crawler.py @@ -5,6 +5,7 @@ from web.signals.website_data_source_crawling_was_completed import website_data_source_crawling_completed from web.models.crawled_pages import CrawledPages from web.models.website_data_sources import WebsiteDataSource +from api.utils.init_vector_store import ensure_vector_database_exists from django.core.files.storage import default_storage from django.core.files.base import ContentFile from django.utils.text import slugify @@ -35,6 +36,10 @@ def start_recursive_crawler(data_source_id, chatbot_id): Exception: If any error occurs during the crawling process, the function will catch the exception, set the crawling status to "failed", and re-raise the exception. """ + # Ensure vector database exists before starting the crawl + + ensure_vector_database_exists(str(chatbot_id)) + # print("Starting recursive crawler") data_source = WebsiteDataSource.objects.get(pk=data_source_id) root_url = data_source.root_url @@ -323,4 +328,4 @@ def crawl(data_source_id, url, crawled_urls, max_pages, chatbot_id): except Exception as e: # Handle other exceptions (e.g., invalid HTML, network issues) and continue crawling logging.exception(f"An unexpected error occurred while crawling URL: {url}") - pass \ No newline at end of file + pass From 61f6611d9844fd8f434fa9aed240da5dcf02f7a2 Mon Sep 17 00:00:00 2001 From: lvalics Date: Wed, 14 Feb 2024 18:50:14 +0000 Subject: [PATCH 7/7] changelog --- dj_backend_server/CHANGELOG.MD | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 dj_backend_server/CHANGELOG.MD diff --git a/dj_backend_server/CHANGELOG.MD b/dj_backend_server/CHANGELOG.MD new file mode 100644 index 00000000..15b70c6c --- /dev/null +++ b/dj_backend_server/CHANGELOG.MD @@ -0,0 +1,6 @@ +2.14.2024 +- Added example.env to streamline environment setup. +- Implemented translation fixes to enhance application localization. +- Updated docker-compose.yaml to prefix each container with oc_ for better namespace management. +- Performed fixes in requirements.txt for improved dependency resolution. +- Ensured existence of Vector Database (QDrant) prior to web crawling operations to address issues encountered with large websites, ensuring Vector Database creation and availability. \ No newline at end of file