added async check by celery

.github/workflows/pyci.yml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       matrix:
         # we want to test our package on several versions of Python
-        python-version: [3.9, 3.11]
+        python-version: [3.11]
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}

Makefile

@@ -24,6 +24,6 @@ selfcheck:
 	poetry check
 
 check:
-	 make lint
+	 lint test
 
-.PHONY: install lint selfcheck check build
+.PHONY: install test lint selfcheck check build

page_analyzer/app.py

@@ -4,6 +4,7 @@
 from urllib.parse import urlparse
 from page_analyzer.url_validator import validate
 from page_analyzer.html_parser import parse_page
+from page_analyzer.tasks import async_check_all_urls
 from flask import (
     Flask,
     render_template,
@@ -78,8 +79,16 @@ def add_url():
 @app.get('/urls')
 def show_all_urls():
     all_urls = get_urls_with_latest_check()
+    message = get_flashed_messages(with_categories=True)
+    return render_template('urls.html', all_urls=all_urls, message=message)
+
+
+@app.post('/urls/checks')
+def check_all_urls():
+    async_check_all_urls.delay()
+    flash('Процесс проверки всех страниц запушен', 'success')
 
-    return render_template('urls.html', all_urls=all_urls)
+    return redirect(url_for('show_all_urls'))
 
 
 @app.get('/urls/<int:id>')

page_analyzer/celery_config.py

@@ -0,0 +1,19 @@
+import os
+from celery import Celery
+from dotenv import load_dotenv
+
+load_dotenv()
+CELERY_BROKER_URL = os.getenv('REDIS_URL')
+CELERY_RESULT_BACKEND = os.getenv('REDIS_URL')
+
+
+def make_celery():
+    celery = Celery(
+        main='page_analyzer',
+        broker=CELERY_BROKER_URL,
+        backend=CELERY_RESULT_BACKEND
+    )
+    return celery
+
+
+celery_app = make_celery()

page_analyzer/db.py

@@ -17,7 +17,6 @@ def fetch_all(connection, query, values=()):
         cur.execute(query, values)
 
         data = cur.fetchall()
-
         connection.commit()
         connection.close()
     return data

page_analyzer/tasks.py

@@ -0,0 +1,22 @@
+from page_analyzer.celery_config import celery_app
+from page_analyzer.db import get_urls_with_latest_check, add_check_to_db
+from page_analyzer.html_parser import parse_page
+import requests
+
+
+@celery_app.task
+def async_check_all_urls():
+    all_urls = get_urls_with_latest_check()
+    for url in all_urls:
+        try:
+            response = requests.get(url.name)
+        except requests.exceptions.RequestException:
+            add_check_to_db(
+                url.id,
+                0,
+                {'title': '', 'h1': '', 'description': 'Ошибка сети'}
+            )
+            continue
+        page_data = parse_page(response.text)
+        status_code = response.status_code
+        add_check_to_db(url.id, status_code, page_data)

page_analyzer/templates/urls.html

@@ -1,7 +1,15 @@
 {% extends "layout.html" %}
+{% block sidebar %}
+{% if message %}
+<div class="alert alert-{{ message[0][0] }} d-flex" role="alert">{{ message[0][1] }}</div>
+{% endif %}
+{% endblock sidebar %}
 {% block content %}
 <h1>Сайты</h1>
-<div class="table-responsive">
+<form method="post" action="{{ url_for('check_all_urls') }}">
+  <input type="submit" class="btn btn-primary" value="Запустить проверку всех сайтов">
+</form>
+<div class="table-responsive mt-2">
     <table class="table table-bordered table-hover text-nowrap" data-test="urls">
         <thead>
         <tr>

tests/test_page_analyzer.py

@@ -95,11 +95,6 @@ def test_get_url(client):
     assert 'https://right1.com' in text
 
 
-@pytest.fixture
-def url():
-    return 'https://python-project-83-rael.onrender.com'
-
-
 def test_url_added(page: Page, url):
     page.goto(url, timeout=15000)
     page.locator('input[name="url"]').type('http://right.com')