Merge pull request #530 from the-hideout/scanner-api

Separate Scanner API functions from interfaces
the-hideout · Jun 7, 2024 · ec8242c · ec8242c
2 parents 46b611b + 1228103
commit ec8242c
Show file tree

Hide file tree

Showing 22 changed files with 2,767 additions and 1,422 deletions.
diff --git a/src/tarkov-data-manager/data/manual_presets.json b/src/tarkov-data-manager/data/manual_presets.json
diff --git a/src/tarkov-data-manager/data/missing_quests.json b/src/tarkov-data-manager/data/missing_quests.json
@@ -1633,5 +1633,163 @@
         "experience": 25000,
         "tarkovDataId": null,
         "factionName": "Any"
+    },
+    "66058cb22cee99303f1ba067": {
+        "id": "66058cb22cee99303f1ba067",
+        "name": "Easy Money - Part 1",
+        "trader": "58330581ace78e27b8b10cee",
+        "location_id": null,
+        "locationName": null,
+        "minPlayerLevel": 3,
+        "taskRequirements": [
+            {
+                "task": "657315e270bb0b8dba00cc48",
+                "name": "Burning Rubber",
+                "status": [
+                    "complete"
+                ]
+            }
+        ],
+        "traderRequirements": [],
+        "objectives": [
+            {
+                "id": "664bbad8d5057479ac4b8002",
+                "type": "findQuestItem",
+                "optional": false,
+                "locationNames": [
+                    "Woods"
+                ],
+                "map_ids": [
+                    "5704e3c2d2720bac5b8b4567"
+                ],
+                "item_id": "664a5775f3d3570fba06be64",
+                "item_name": "Bison VS Undertaker poster pack",
+                "count": 1
+            },
+            {
+                "id": "664bbb5f217c767c35ae3d51",
+                "type": "findQuestItem",
+                "optional": false,
+                "locationNames": [
+                    "Woods"
+                ],
+                "map_ids": [
+                    "5704e3c2d2720bac5b8b4567"
+                ],
+                "item_id": "664b69c5a082271bc46c4e11",
+                "item_name": "A pack of Killa and Tagilla posters",
+                "count": 1
+            },
+            {
+                "id": "664bbb73c71d456fd03714ca",
+                "type": "findQuestItem",
+                "optional": false,
+                "locationNames": [
+                    "Woods"
+                ],
+                "map_ids": [
+                    "5704e3c2d2720bac5b8b4567"
+                ],
+                "item_id": "664b69e8e1238e506d3630af",
+                "item_name": "A pack of 'Easy money on betting' posters",
+                "count": 1
+            },
+            {
+                "id": "6606b709f7ce34bb26221cab",
+                "type": "plantQuestItem",
+                "count": 1,
+                "optional": false,
+                "locationNames": [
+                    "Customs"
+                ],
+                "map_ids": [
+                    "56f40101d2720b2a4d8b45d6"
+                ],
+                "zoneKeys": [
+                    "place_flyers1"
+                ],
+                "zoneNames": [],
+                "item_id": "664a5775f3d3570fba06be64"
+            },
+            {
+                "id": "6606b74ebce4c3e0bda438fd",
+                "type": "plantQuestItem",
+                "count": 1,
+                "optional": false,
+                "locationNames": [
+                    "Reserve"
+                ],
+                "map_ids": [
+                    "5704e5fad2720bc05b8b4567"
+                ],
+                "zoneKeys": [
+                    "place_flaers2"
+                ],
+                "zoneNames": [],
+                "item_id": "664b69c5a082271bc46c4e11"
+            },
+            {
+                "id": "6606b752a6c704c4333a804d",
+                "type": "plantQuestItem",
+                "count": 1,
+                "optional": false,
+                "locationNames": [
+                    "Lighthouse"
+                ],
+                "map_ids": [
+                    "5704e4dad2720bb55b8b4567"
+                ],
+                "zoneKeys": [
+                    "place_flyers4"
+                ],
+                "zoneNames": [],
+                "item_id": "664b69e8e1238e506d3630af"
+            }
+        ],
+        "startRewards": {
+            "traderStanding": [],
+            "items": [],
+            "offerUnlock": [],
+            "skillLevelReward": [],
+            "traderUnlock": []
+        },
+        "finishRewards": {
+            "traderStanding": [
+                {
+                    "trader_id": "54cb50c76803fa8b248b4571",
+                    "name": "Prapor",
+                    "standing": 0.04
+                }
+            ],
+            "items": [
+                {
+                    "item": "5449016a4bdc2d6f028b456f",
+                    "item_name": "Roubles",
+                    "count": 35000,
+                    "contains": [],
+                    "attributes": []
+                }
+            ],
+            "offerUnlock": [],
+            "skillLevelReward": [],
+            "traderUnlock": [
+                {
+                    "trader_id": "6617beeaa9cfa777ca915b7c",
+                    "trader_name": "Ref"
+                }
+            ]
+        },
+        "failureOutcome": {
+            "traderStanding": [],
+            "items": [],
+            "offerUnlock": [],
+            "skillLevelReward": [],
+            "traderUnlock": []
+        },
+        "failConditions": [],
+        "restartable": false,
+        "experience": 4500,
+        "tarkovDataId": null,
+        "factionName": "Any"
     }
 }
diff --git a/src/tarkov-data-manager/index.mjs b/src/tarkov-data-manager/index.mjs
@@ -14,8 +14,8 @@ import remoteData from './modules/remote-data.mjs';
 import jobs from './jobs/index.mjs';
 import {connection, query, format} from './modules/db-connection.mjs';
 import timer from './modules/console-timer.js';
-import { userFlags, scannerFlags, refreshScannerUsers } from './modules/scanner-framework.mjs';
-import scannerApi from './modules/scanner-api.mjs';
+import { userFlags, scannerFlags, refreshScannerUsers } from './modules/scanner-api.mjs';
+import scannerHttpApi from './modules/scanner-http-api.mjs';
 import webhookApi from './modules/webhook-api.mjs';
 import publicApi from './modules/public-api.mjs';
 import { uploadToS3, getImages, getLocalBucketContents, addFileToBucket, deleteFromBucket, renameFile, copyFile } from './modules/upload-s3.mjs';
@@ -1784,7 +1784,7 @@ app.post('/wipes', async (req, res) => {
         console.log(`creating wipe: ${req.body.start_date} ${req.body.version}`);
         const result = await query('INSERT INTO wipe (start_date, version) VALUES (?, ?)', [req.body.start_date, req.body.version]);
         let lastPriceId = 0;
-        const lastPrice = await query('SELECT id FROM price_data WHERE pve=0 ORDER BY id DESC LIMIT 1');
+        const lastPrice = await query('SELECT id FROM price_data WHERE game_mode = 0 ORDER BY id DESC LIMIT 1');
         if (lastPrice.length > 0) {
             lastPriceId = lastPrice[0].id;
         }
@@ -1824,7 +1824,7 @@ app.delete('/wipes/:id', async (req, res) => {
 });
 
 app.all('/api/scanner/:resource', async (req, res) => {
-    scannerApi.request(req, res, req.params.resource);
+    scannerHttpApi.request(req, res, req.params.resource);
 });
 
 app.post('/api/webhooks/:hooksource/:webhookid/:webhookkey', async (req, res) => {

diff --git a/src/tarkov-data-manager/jobs/archive-prices.mjs b/src/tarkov-data-manager/jobs/archive-prices.mjs
@@ -23,49 +23,49 @@ class ArchivePricesJob extends DataJob {
 
         // archive max_days_per_run number of days
         for (let i = 0; i < max_days_per_run; i++) {
-            for (let pve = 0; pve < 2; pve++) {
-                const gameMode = pve ? 'pve' : 'regular';
+            for (let gameMode = 0; gameMode < 2; gameMode++) {
+                const gameModeName = gameMode ? 'pve' : 'regular';
                 // get the price with the oldest timestamp
                 const oldestPrice = await this.query(`
                     SELECT * FROM price_data 
-                    WHERE timestamp < ? AND pve = ?
+                    WHERE timestamp < ? AND game_mode = ?
                     ORDER BY timestamp
                     LIMIT 1
-                `, [cutoff, pve]);
+                `, [cutoff, gameMode]);
                 if (oldestPrice.length === 0) {
-                    this.logger.success(`No ${gameMode} prices found before ${cutoff}`);
+                    this.logger.success(`No ${gameModeName} prices found before ${cutoff}`);
                     return;
                 }
                 // convert oldest price date to YYYY-MM-dd
                 const archiveDate = this.getMysqlDate(oldestPrice[0].timestamp);
 
-                this.logger.log(`Archiving ${gameMode} prices for ${archiveDate}`);
+                this.logger.log(`Archiving ${gameModeName} prices for ${archiveDate}`);
 
                 // get minimum and average prices per item during day
                 const itemPrices = await this.query(`
                     SELECT item_id, MIN(price) as min_price, AVG(price) as avg_price 
                     FROM price_data 
-                    WHERE timestamp >= ? AND timestamp < ? + INTERVAL 1 DAY AND pve = ?
+                    WHERE timestamp >= ? AND timestamp < ? + INTERVAL 1 DAY AND game_mode = ?
                     GROUP BY item_id
-                `, [archiveDate, archiveDate, pve]);
+                `, [archiveDate, archiveDate, gameMode]);
 
                 // add min and average prices to price archive insert
                 const insertValues = [];
                 for (const itemPrice of itemPrices) {
-                    insertValues.push(itemPrice.item_id, archiveDate, itemPrice.min_price, Math.round(itemPrice.avg_price), pve);
+                    insertValues.push(itemPrice.item_id, archiveDate, itemPrice.min_price, Math.round(itemPrice.avg_price), gameMode);
                 }
 
                 // insert archived prices
                 const insertStart = new Date();
                 await this.query(`
                     INSERT INTO price_archive
-                        (item_id, price_date, price_min, price_avg, pve)
+                        (item_id, price_date, price_min, price_avg, game_mode)
                     VALUES
                         ${Object.keys(itemPrices).map(() => '(?, ?, ?, ?, ?)').join(', ')}
                     ON DUPLICATE KEY UPDATE
                         price_min=VALUES(price_min), price_avg=VALUES(price_avg)
                 `, insertValues);
-                this.logger.log(`Inserted ${Object.keys(itemPrices).length} ${gameMode} archived prices in ${new Date() - insertStart}ms`);
+                this.logger.log(`Inserted ${Object.keys(itemPrices).length} ${gameModeName} archived prices in ${new Date() - insertStart}ms`);
 
                 // delete the prices we just archived
                 await this.deletePricesThrough(archiveDate);
@@ -79,7 +79,7 @@ class ArchivePricesJob extends DataJob {
     }
 
     // deletes all prices through the given YYY-MM-dd date
-    deletePricesThrough = async (mysqlDateCutoff, pve) => {
+    deletePricesThrough = async (mysqlDateCutoff, gameMode) => {
         // delete archived prices from main price table
         // can only delete 100k at a time, so need to loop
         const batchSize = this.maxQueryRows;
@@ -88,9 +88,9 @@ class ArchivePricesJob extends DataJob {
         while (true) {
             const deleteResult = await this.query(`
                 DELETE FROM price_data 
-                WHERE timestamp < ? + INTERVAL 1 DAY AND pve = ?
+                WHERE timestamp < ? + INTERVAL 1 DAY AND game_mode = ?
                 LIMIT ?
-            `, [mysqlDateCutoff, pve, batchSize]);
+            `, [mysqlDateCutoff, gameMode, batchSize]);
             deletedCount += deleteResult.affectedRows;
             if (deleteResult.affectedRows < batchSize) {
                 break;

diff --git a/src/tarkov-data-manager/jobs/check-scanners.mjs b/src/tarkov-data-manager/jobs/check-scanners.mjs
@@ -1,6 +1,6 @@
 import { DateTime } from 'luxon';
 
-import { scannerFlags, userFlags } from '../modules/scanner-framework.mjs';
+import { scannerFlags, userFlags } from '../modules/scanner-api.mjs';
 import tarkovDevData from '../modules/tarkov-dev-data.mjs';
 import DataJob from '../modules/data-job.mjs';
 

diff --git a/src/tarkov-data-manager/jobs/index.mjs b/src/tarkov-data-manager/jobs/index.mjs
@@ -23,8 +23,6 @@ const defaultJobs = {
     'update-game-status': '*/15 * * * *',
     'start-trader-scan': '30 9,21 * * *',
 };
-// Too much memory :'(
-// 'update-longtime-data': '49 8 * * *'
 
 // these jobs only run on the given schedule when not in dev mode
 const nonDevJobs = {};

diff --git a/src/tarkov-data-manager/jobs/start-trader-scan.mjs b/src/tarkov-data-manager/jobs/start-trader-scan.mjs
@@ -12,7 +12,7 @@ class StartTraderScanJob extends DataJob {
         if (await scannerApi.currentTraderScan()) {
             this.logger.log('Trader scan already in progress');
         } else {
-            await scannerApi.startTraderScan({scanner: {id: 0}});
+            await scannerApi.startTraderScan();
         }
 
         //let scanners = webSocketServer.launchedScanners().filter(c => c.settings.scanStatus === 'idle' && c.settings.scanMode === 'auto');

diff --git a/src/tarkov-data-manager/jobs/update-archived-prices.mjs b/src/tarkov-data-manager/jobs/update-archived-prices.mjs
@@ -59,7 +59,7 @@ class UpdateArchivedPricesJob extends DataJob {
                         price_archive
                     WHERE
                         price_date > ? AND
-                        pve = ?
+                        game_mode = ?
                     ORDER BY price_date, item_id
                     LIMIT ?, ?
                 `, [dateCutoff, pve, offset, batchSize]);

diff --git a/src/tarkov-data-manager/jobs/update-barters-wiki.mjs b/src/tarkov-data-manager/jobs/update-barters-wiki.mjs
@@ -2,7 +2,7 @@ import fs from 'node:fs';
 import path from 'node:path';
 
 import got from 'got';
-import cheerio from 'cheerio';
+import * as cheerio from 'cheerio';
 
 import remoteData from '../modules/remote-data.mjs';
 import fixName from '../modules/wiki-replacements.js';

diff --git a/src/tarkov-data-manager/jobs/update-barters.mjs b/src/tarkov-data-manager/jobs/update-barters.mjs
@@ -104,7 +104,7 @@ class UpdateBartersJob extends DataJob {
             }
             const trader = this.traders.find(t => t.id === offer.trader_id);
             const questUnlock = this.getQuestUnlock(offer);
-            const assort = this.traderAssorts[trader.id].find(assort => assort.id === offer.id);
+            const assort = this.traderAssorts[trader.id]?.find(assort => assort.id === offer.id);
             const barter = {
                 id: offer.id,
                 trader_id: trader.id,

diff --git a/src/tarkov-data-manager/jobs/update-historical-prices.mjs b/src/tarkov-data-manager/jobs/update-historical-prices.mjs
@@ -62,7 +62,7 @@ class UpdateHistoricalPricesJob extends DataJob {
                         price_data
                     WHERE
                         timestamp > ? AND
-                        pve = ?
+                        game_mode = ?
                     GROUP BY item_id, timestamp
                     ORDER BY timestamp, item_id
                     LIMIT ?, ?