Merge pull request #4 from Jeffrey04/20240910-log-requests

20240910 log requests

podman/backend-migrate/migrate.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+export PATH="/root/.local/bin:${PATH}"
+
+# install pipx
+apt-get update && apt-get install --no-install-suggests --no-install-recommends --yes pipx
+
+# install poetry
+pipx install poetry
+# poetry install
+poetry install --only=main
+
+poetry run python manage.py migrate

pyproject.toml

@@ -12,6 +12,7 @@ django = "^5.1.1"
 django-structlog = "^8.1.0"
 django-dotenv = "^1.4.2"
 gunicorn = "^23.0.0"
+psycopg2-binary = "^2.9.9"
 
 
 [tool.poetry.group.dev.dependencies]

scripts/migrate.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+NETWORK="${DOCKER_NETWORK:-blockedornot}"
+
+podman run \
+    --rm  -it \
+    --env-file .env \
+    --mount type=bind,source="$(pwd)",target=/project \
+    --workdir /project \
+    --network "$NETWORK" \
+    python:3.11-slim bash podman/backend-migrate/migrate.sh

src/api/migrations/0001_initial.py

@@ -0,0 +1,31 @@
+# Generated by Django 5.1.1 on 2024-09-10 14:13
+
+import datetime
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Query',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('query', models.CharField()),
+                ('query_cleaned', models.CharField()),
+                ('dns_public_result', models.CharField()),
+                ('dns_public', models.CharField()),
+                ('dns_isp_result', models.CharField()),
+                ('dns_isp', models.CharField()),
+                ('blocked', models.BooleanField()),
+                ('different_ip', models.BooleanField()),
+                ('measurement_url', models.CharField(blank=True)),
+                ('creation_time', models.DateTimeField(default=datetime.datetime(2024, 9, 10, 14, 13, 22, 498677))),
+            ],
+        ),
+    ]

src/api/models.py

@@ -1,3 +1,20 @@
+from datetime import datetime
+
 from django.db import models
 
-# Create your models here.
+
+class Query(models.Model):
+    query = models.CharField()
+    query_cleaned = models.CharField()
+
+    dns_public_result = models.CharField()
+    dns_public = models.CharField()
+
+    dns_isp_result = models.CharField()
+    dns_isp = models.CharField()
+
+    blocked = models.BooleanField()
+    different_ip = models.BooleanField()
+    measurement_url = models.CharField(blank=True)
+
+    creation_time = models.DateTimeField(default=datetime.now())

src/api/views.py

@@ -1,9 +1,7 @@
-import json
 import subprocess
 from concurrent.futures import ProcessPoolExecutor
-from functools import partial
 from os import environ
-from typing import Tuple
+from typing import NamedTuple
 from urllib.parse import urlparse
 
 import structlog
@@ -16,12 +14,21 @@
 )
 from django.views.decorators.http import require_GET
 
+from api.models import Query
+
 DEFAULT_DNS_PUBLIC = "1.1.1.1"
 DEFAULT_DNS_ISP = "183.171.212.193"
 DEFAULT_SINKHOLE_IP = "175.139.142.25"
 
 logger = structlog.get_logger(__name__)
 
+class DNSCheck(NamedTuple):
+    blocked: bool
+    different_ip: bool
+
+    dns_isp_result: str
+    dns_public_result: str
+
 
 @require_GET
 def query(request: HttpRequest) -> HttpResponse:
@@ -31,19 +38,33 @@ def query(request: HttpRequest) -> HttpResponse:
 
     if query:
         try:
-            is_blocked, different_ip = _doge_check_is_blocked(query)
+            dns_result = _doge_check_is_blocked(query)
+            measurement = (
+                _ooni_check_url(query)
+                if dns_result.blocked or dns_result.different_ip
+                else None
+            )
             result = JsonResponse(
                 dict(
                     {
-                        "blocked": is_blocked,
-                        "different_ip": different_ip,
-                        "measurement": _ooni_check_url(query)
-                        if is_blocked or different_ip
-                        else None,
+                        "blocked": dns_result.blocked,
+                        "different_ip": dns_result.different_ip,
+                        "measurement": measurement,
                     },
                     query=query,
                 )
             )
+            Query.objects.create(
+                query=request.GET.get("query"),
+                query_cleaned=query,
+                dns_public_result=dns_result.dns_public_result,
+                dns_public=environ.get("DNS_PUBLIC", DEFAULT_DNS_PUBLIC),
+                dns_isp_result=dns_result.dns_isp_result,
+                dns_isp=environ.get("DNS_ISP", DEFAULT_DNS_ISP),
+                blocked=dns_result.blocked,
+                different_ip=dns_result.different_ip,
+                measurement_url=measurement or "",
+            )
 
         except Exception:
             result = HttpResponseServerError()
@@ -81,7 +102,7 @@ def _doge_check_against_dns(query: str, dns: str) -> subprocess.CompletedProcess
     return process
 
 
-def _doge_check_is_blocked(query: str) -> Tuple[bool, bool]:
+def _doge_check_is_blocked(query: str) -> DNSCheck:
     with ProcessPoolExecutor() as executor:
         futures = (
             executor.submit(
@@ -96,15 +117,18 @@ def _doge_check_is_blocked(query: str) -> Tuple[bool, bool]:
             ),
         )
 
-        result = tuple(
-            set(future.result(timeout=5).stdout.strip().splitlines())
-            for future in futures
-        )
+        results = tuple(future.result(timeout=5).stdout.strip() for future in futures)
+        result_set = tuple(set(result.splitlines()) for result in results)
 
-        return any(
-            ip.startswith(environ.get("SINKHOLE_IP", DEFAULT_SINKHOLE_IP))
-            for ip in result[1]
-        ), len(result[0].intersection(result[1])) == 0
+        return DNSCheck(
+            blocked=any(
+                ip.startswith(environ.get("SINKHOLE_IP", DEFAULT_SINKHOLE_IP))
+                for ip in result_set[1]
+            ),
+            different_ip=len(result_set[0].intersection(result_set[1])) == 0,
+            dns_isp_result=results[0],
+            dns_public_result=results[1],
+        )
 
 
 def _ooni_check_url(query: str) -> str:

src/blockedornot/settings.py

@@ -39,6 +39,7 @@
     # "django.contrib.messages",
     # "django.contrib.staticfiles",
     "django_structlog",
+    "api",
 ]
 
 MIDDLEWARE = [
@@ -78,8 +79,12 @@
 
 DATABASES = {
     "default": {
-        "ENGINE": "django.db.backends.sqlite3",
-        "NAME": BASE_DIR / "db.sqlite3",
+        "ENGINE": "django.db.backends.postgresql",
+        "NAME": environ.get("POSTGRES_DB", "legisweb"),
+        "USER": environ.get("POSTGRES_USER", "legisweb"),
+        "PASSWORD": environ.get("POSTGRES_PASSWORD", "abc123"),
+        "HOST": environ.get("DATABASE_HOST", "localhost"),
+        "PORT": environ.get("DATABASE_PORT", "5432"),
     }
 }
 

src/frontend/src/routes/root.tsx

@@ -111,16 +111,17 @@ function ResultExplanation({ result }) {
 	if (result.blocked) {
 		text = (
 			<p>
-				A well-known IP hosted by the government for blocked websites is
-				returned by our ISP/Telco DNS.
+				Instead of the intended IP address, this URL is being directed by TMNet
+				(AS4788) DNS server to the IP address 175.139.142.25 for websites
+				blocked by MCMC.
 			</p>
 		);
 	} else if (result.blocked === false && result.different_ip) {
 		text = (
 			<p>
-				Different sets of IPs were received from our ISP/Telco and a well-known
-				public DNS provider. It could be due to the use of Content Delivery
-				Network by the website operator.
+				Different sets of IPs were received from our TMNet (AS4788) and a
+				well-known public DNS provider. It could be due to the use of Content
+				Delivery Network by the website operator.
 			</p>
 		);
 	}