Merge pull request #870 from CodeForAfrica/outline-vpn-user-stats

@Outlinevpn Process Daily usage statistics

apps/vpnmanager/package.json

@@ -27,6 +27,7 @@
     "@sentry/nextjs": "catalog:",
     "@svgr/webpack": "catalog:",
     "@types/jest": "catalog:",
+    "better-sqlite3": "catalog:",
     "googleapis": "catalog:",
     "jest": "catalog:",
     "next": "catalog:",

apps/vpnmanager/src/lib/data/database.ts

@@ -0,0 +1,131 @@
+import betterSqlite3 from "better-sqlite3";
+import path from "path";
+
+const dbPath = path.resolve(process.cwd(), "data", "database.sqlite");
+const db = betterSqlite3(dbPath);
+
+export interface Record {
+  ID?: number;
+  userId: string;
+  usage: number;
+  date: string;
+  cumulativeData: number;
+  email: string;
+  createdAt: string;
+}
+
+export interface Filters {
+  email?: string;
+  date?: string | { start: string; end: string };
+  userId?: string;
+  ID?: number;
+  groupBy?: "email" | "date";
+  orderBy?: string;
+}
+
+class Model {
+  static initialize() {
+    const createTable = `
+      CREATE TABLE IF NOT EXISTS records (
+        ID INTEGER PRIMARY KEY AUTOINCREMENT,
+        userId TEXT NOT NULL,
+        usage INTEGER NOT NULL,
+        date TEXT NOT NULL,
+        cumulativeData INTEGER NOT NULL,
+        email TEXT NOT NULL,
+        createdAt TEXT NOT NULL,
+        UNIQUE (date, userId)
+      )
+    `;
+    db.exec(createTable);
+  }
+
+  static createOrUpdate(record: Record) {
+    const insertData = db.prepare(`
+      INSERT INTO records (userId, usage, date, cumulativeData, email, createdAt)
+      VALUES (?, ?, ?, ?, ?, ?)
+      ON CONFLICT(date, userId)
+      DO UPDATE SET
+        usage = excluded.usage,
+        cumulativeData = excluded.cumulativeData,
+        email = excluded.email,
+        createdAt = excluded.createdAt;
+    `);
+    const info = insertData.run(
+      record.userId,
+      record.usage,
+      record.date,
+      record.cumulativeData,
+      record.email,
+      record.createdAt,
+    );
+    return { ...record, ID: info.lastInsertRowid };
+  }
+
+  static update(ID: number, updates: Partial<Record>) {
+    const setClause = Object.keys(updates)
+      .map((key) => `${key} = ?`)
+      .join(", ");
+    const query = `UPDATE records SET ${setClause} WHERE ID = ?`;
+    const stmt = db.prepare(query);
+    return stmt.run([...Object.values(updates), ID]);
+  }
+
+  static delete(ID: number) {
+    const stmt = db.prepare("DELETE FROM records WHERE ID = ?");
+    return stmt.run(ID);
+  }
+
+  static getAll(filters: Filters = {}) {
+    let query = "SELECT";
+    const params: any[] = [];
+    if (filters.groupBy === "email" || filters.groupBy === "date") {
+      query +=
+        filters.groupBy === "email"
+          ? " email, userId, SUM(usage) as totalUsage FROM records"
+          : " date, SUM(usage) as totalUsage FROM records";
+    } else {
+      query += " * FROM records";
+    }
+    query += " WHERE 1=1";
+    if (filters.email) {
+      query += " AND email = ?";
+      params.push(filters.email);
+    }
+    if (filters.date) {
+      if (typeof filters.date === "string") {
+        query += " AND date = ?";
+        params.push(filters.date);
+      } else {
+        query += " AND date BETWEEN ? AND ?";
+        params.push(filters.date.start, filters.date.end);
+      }
+    }
+    if (filters.userId) {
+      query += " AND userId = ?";
+      params.push(filters.userId);
+    }
+    if (filters.ID) {
+      query += " AND ID = ?";
+      params.push(filters.ID);
+    }
+
+    if (filters.groupBy) {
+      if (filters.groupBy === "email") {
+        query += " GROUP BY email";
+      } else if (filters.groupBy === "date") {
+        query += " GROUP BY date";
+      }
+    }
+    if (filters.orderBy) {
+      query += ` ORDER BY ${filters.orderBy}`;
+    }
+    const stmt = db.prepare(query);
+    return stmt.all(params);
+  }
+}
+
+// Initialize the database
+Model.initialize();
+
+export { Model };

apps/vpnmanager/src/lib/statistics.ts

@@ -0,0 +1,51 @@
+import { NextApiRequest } from "next/types";
+import { OutlineVPN } from "./outline";
+import { Filters, Model, Record } from "@/vpnmanager/lib/data/database";
+
+const vpnManager = new OutlineVPN({
+  apiUrl: process.env.NEXT_APP_VPN_API_URL as string,
+});
+
+export async function processUserStats() {
+  const date = `${new Date().getFullYear()}-${new Date().getMonth() + 1}-${new Date().getDate()}`;
+  const { bytesTransferredByUserId = {} } = await vpnManager.getDataUsage();
+  const allUsers = await vpnManager.getUsers();
+  const unprocessedUsers: Omit<Record, "ID" | "createdAt">[] = Object.keys(
+    bytesTransferredByUserId,
+  ).map((key: string) => {
+    const userDetails = allUsers.find(({ id }) => id === key);
+    const newData = {
+      userId: key,
+      usage: Math.ceil(bytesTransferredByUserId[key] / 30),
+      date,
+      cumulativeData: bytesTransferredByUserId[key],
+      email: userDetails?.name || "",
+    };
+    Model.createOrUpdate({ ...newData, createdAt: new Date().toISOString() });
+    return newData;
+  });
+  return unprocessedUsers;
+}
+
+export async function getStats(req: NextApiRequest) {
+  const filters: Partial<Filters> & {
+    "date.start"?: string;
+    "date.end"?: string;
+  } = req.query;
+  const validFilters = {
+    email: filters.email,
+    ID: filters.ID,
+    userId: filters.userId,
+    groupBy: filters.groupBy as "email" | "date",
+    orderBy: filters.orderBy,
+    date:
+      filters["date.start"] && filters["date.end"]
+        ? {
+            start: filters["date.start"],
+            end: filters["date.end"],
+          }
+        : filters.date,
+  };
+
+  return Model.getAll(validFilters);
+}

apps/vpnmanager/src/middleware.ts

@@ -0,0 +1,17 @@
+import type { NextRequest } from "next/server";
+
+// Limit the middleware to paths starting with `/api/`
+export const config = {
+  matcher: "/api/:function*",
+};
+
+export function middleware(req: NextRequest) {
+  const key: string = req.headers.get("x-api-key") as string;
+  const API_SECRET_KEY = process.env.API_SECRET_KEY;
+  if (!(key && key === API_SECRET_KEY)) {
+    return Response.json(
+      { success: false, message: "INVALID_API_KEY" },
+      { status: 403 },
+    );
+  }
+}

apps/vpnmanager/src/pages/api/processGsheet.ts

@@ -3,13 +3,10 @@ import { processNewUsers } from "@/vpnmanager/lib/processUsers";
 
 export async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    const key: string = req.headers["x-api-key"] as string;
-    const API_SECRET_KEY = process.env.API_SECRET_KEY;
-    if (!(key && key !== API_SECRET_KEY)) {
-      return res.status(403).json({ message: "INVALID_API_KEY" });
-    }
     processNewUsers();
     return res.status(200).json({ message: "Process Started" });
-  } catch (error) {}
+  } catch (error) {
+    return res.status(500).json(error);
+  }
 }
 export default handler;

apps/vpnmanager/src/pages/api/statistics.ts

@@ -0,0 +1,22 @@
+import { NextApiResponse, NextApiRequest } from "next";
+import { processUserStats, getStats } from "@/vpnmanager/lib/statistics";
+import { RestMethodFunctions, RestMethods } from "@/vpnmanager/types";
+
+const methodToFunction: RestMethodFunctions = {
+  POST: processUserStats,
+  GET: getStats,
+};
+
+export async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    const statFunc = methodToFunction[req.method as RestMethods];
+    if (!statFunc) {
+      return res.status(404).json({ message: "Requested path not found" });
+    }
+    const data = await statFunc(req);
+    return res.status(200).json(data);
+  } catch (error) {
+    return res.status(500).json(error);
+  }
+}
+export default handler;

apps/vpnmanager/src/types.d.ts

@@ -1,3 +1,5 @@
+import { NextApiRequest } from "next";
+
 export interface OutlineOptions {
   apiUrl: string;
   fingerprint?: string;
@@ -39,3 +41,9 @@ export interface SheetRow {
   endDate: string;
   keySent: "Yes" | "No";
 }
+
+export type RestMethods = "GET" | "POST" | "PUT" | "DELETE" | "PATCH";
+
+export type RestMethodFunctions = {
+  [K in RestMethods]?: (req: NextApiRequest) => Promise<any>;
+};

docker-compose.yml

@@ -175,7 +175,11 @@ services:
         - API_SECRET_KEY
     environment:
       NODE_ENV: ${NODE_ENV:-production}
+      NODE_TLS_REJECT_UNAUTHORIZED: 0
+      NEXT_APP_VPN_API_URL: ${NEXT_APP_VPN_API_URL}
     ports:
       - ${VPN_MANAGER_PORT:-3000}:3000
+    volumes:
+      - ./db_data:/apps/vpnmanager/data
 volumes:
   db_data:

packages/hurumap-next/src/Map/Layers.js

@@ -223,7 +223,7 @@ function Layers({
           });
         }
       } else {
-        const mark = new L.Marker(layer.getBounds().getCenter(), {
+        const mark = new L.Marker(layer.getBounds()?.getCenter(), {
           icon: pinIcon,
         });
         mark.on("click", () => {

pnpm-workspace.yaml

@@ -79,6 +79,7 @@ catalog:
   babel-jest: ^29.7.0
   babel-loader: ^9.1.3
   babel-plugin-transform-imports: ^2.0.0
+  better-sqlite3: "^11.2.1"
   camelcase-keys: ^9.1.3
   clsx: ^2.1.1
   crawler-user-agents: ^1.0.146