add job summary alerts

src/tarkov-data-manager/data/changed_quests.json

@@ -1290,42 +1290,6 @@
             ]
         }
     },
-    "_63967028c4a91c5cb76abd81": {
-        "name": "Trouble in the Big City",
-        "finishRewardsChanged": {
-            "traderStanding": [
-                {
-                    "trader_id": "638f541a29ffd1183d187f57",
-                    "name": "Lightkeeper",
-                    "standing": 0.05
-                }
-            ],
-            "offerUnlock": [
-                {
-                    "id": "63a2359ef194393ecf632fa2",
-                    "trader_id": "5a7c2eca46aef81a7ca2145d",
-                    "trader_name": "Mechanic",
-                    "level": 4,
-                    "item": "6389c85357baa773a825b356",
-                    "item_name": "Advanced current converter",
-                    "count": 1,
-                    "contains": [],
-                    "attributes": []
-                },
-                {
-                    "id": "012345678901234567890123",
-                    "trader_id": "5935c25fb3acc3127c3d8cd9",
-                    "trader_name": "Peacekeeper",
-                    "level": 4,
-                    "item": "657eb3773271d8578610fe28",
-                    "item_name": "SIG MCX SPEAR 6.8x51 assault rifle Default",
-                    "count": 1,
-                    "contains": [],
-                    "attributes": []
-                }
-            ]
-        }
-    },
     "59675d6c86f7740a842fc482": {
         "name": "Ice Cream Cones",
         "objectivePropertiesChanged": {
@@ -1409,7 +1373,7 @@
     "5b47891f86f7744d1b23c571": {
         "name": "Living High Is Not a Crime - Part 1",
         "finishRewardsAdded": {
-        "offerUnlock": [
+            "offerUnlock": [
                 {
                     "id": "66c46c36f7ca8a539f08600d5",
                     "trader_id": "6617beeaa9cfa777ca915b7c",
@@ -1423,5 +1387,70 @@
                 }
             ]
         }
+    },
+    "63966fd9ea19ac7ed845db30": {
+        "name": "Payback",
+        "finishRewardsChanged": {
+            "offerUnlock": [
+                {
+                    "id": "66c46c3ef7ca8a539f0866055",
+                    "trader_id": "5ac3b934156ae10c4430e83c",
+                    "trader_name": "Ragman",
+                    "level": 4,
+                    "item": "5b4329f05acfc47a86086aa1",
+                    "item_name": "DevTac Ronin Respirator",
+                    "count": 1,
+                    "contains": [],
+                    "attributes": []
+                }
+            ]
+        }
+    },
+    "63967028c4a91c5cb76abd81": {
+        "name": "Trouble in the Big City",
+        "finishRewardsAdded": {
+            "offerUnlock": [
+                {
+                    "id": "66c46c7af7ca8a539f0881693",
+                    "trader_id": "5935c25fb3acc3127c3d8cd9",
+                    "trader_name": "Peacekeeper",
+                    "level": 4,
+                    "item": "65293c7a17e14363030ad308",
+                    "item_name": "AR-10 7.62x51 Lancer L7AWM 25-round magazine",
+                    "count": 1,
+                    "contains": [],
+                    "attributes": []
+                },
+                {
+                    "id": "66c46c7df7ca8a539f0882ab2",
+                    "trader_id": "5935c25fb3acc3127c3d8cd9",
+                    "trader_name": "Peacekeeper",
+                    "level": 4,
+                    "item": "6529243824cbe3c74a05e5c1",
+                    "item_name": "6.8x51mm SIG Hybrid",
+                    "count": 1,
+                    "contains": [],
+                    "attributes": []
+                }
+            ]
+        }
+    },
+    "63966fccac6f8f3c677b9d89": {
+        "name": "Snatch",
+        "finishRewardsChanged": {
+            "offerUnlock": [
+                {
+                    "id": "66c46c3ef7ca8a539f0865fb5",
+                    "trader_id": "5ac3b934156ae10c4430e83c",
+                    "trader_name": "Ragman",
+                    "level": 4,
+                    "item": "657089638db3adca1009f4ca",
+                    "item_name": "Atomic Defense CQCM ballistic mask (Black)",
+                    "count": 1,
+                    "contains": [],
+                    "attributes": []
+                }
+            ]
+        }
     }
 }

src/tarkov-data-manager/jobs/index.mjs

@@ -13,7 +13,7 @@ const defaultJobs = {
     'update-quests': '3-59/10 * * * *',
     'update-maps': '4-59/10 * * * *',
     'check-scanners': '6,36 * * * *',
-    'update-td-data': '7-59/10 * * * *',
+    //'update-td-data': '7-59/10 * * * *',
     'archive-prices': '38 0,12 * * *',
     'verify-wiki': '7 9 * * *',
     'update-trader-assorts': 'traderScansEnded',

src/tarkov-data-manager/jobs/update-new-items.mjs

@@ -87,7 +87,6 @@ class UpdateNewItemsJob extends DataJob {
         });
 
         const foundItems = [];
-        const addedItems = [];
 
         const doNotUse = /DO[ _]NOT[ _]USE|translation_pending/;
         for (const item of items) {
@@ -119,7 +118,7 @@ class UpdateNewItemsJob extends DataJob {
                 });
 
                 console.log(`${name} added`);
-                addedItems.push(`${name} ${item._id}`);
+                this.addJobSummary(`${name} ${item._id}`, 'Added Items');
 
                 if (item._props.QuestItem){
                     await remoteData.addType(item._id, 'quest')
@@ -132,14 +131,6 @@ class UpdateNewItemsJob extends DataJob {
             }
         }
 
-        if (addedItems.length > 0) {
-            this.discordAlert({
-                title: 'New item(s) added',
-                message: addedItems.join('\n'),
-            });
-        }
-
-        const removedItems = [];
         for (const itemId of currentItems.keys()){
             if (bsgData[itemId]) {
                 continue;
@@ -151,14 +142,8 @@ class UpdateNewItemsJob extends DataJob {
             if (item.types.includes('disabled')) {
                 continue;
             }
-            removedItems.push(item);
             this.logger.warn(`${item.name} (${item.id}) is no longer available in the game`);
-        }
-        if (removedItems.length > 0) {
-            this.discordAlert({
-                title: `Item(s) removed from game`,
-                message: removedItems.map(item => `${item.name} ${item.id}`).join('\n'),
-            });
+            this.addJobSummary(`${item.name} ${item.id}`, 'Removed Items');
         }
 
         this.logger.succeed('New item check complete');

src/tarkov-data-manager/jobs/update-tc-data.mjs

@@ -12,23 +12,19 @@ class UpdateTcDataJob extends DataJob {
         this.logger.time('tc-download');
         const returnValue = {};
         for (const gameMode of gameModes) {
+            this.logger.log(`Downloading ${gameMode.name} data...`);
             returnValue[gameMode.name] = await tarkovChanges.downloadAll({returnPartial: true, gameMode: gameMode.name}).then(results => {
-                if (Object.keys(results) > 1) {
-                    this.logger.log(`Downloaded ${Object.keys(results).filter(key => key !== 'errors').join(', ')} ${gameMode.name} files`);
-                }
-                if (results.errors.length === 0) {
-                    this.logger.success(`Successfully downloaded ${gameMode.name} data from Tarkov Changes`);
+                if (Object.keys(results).length > 1) {
+                    this.logger.success(`Downloaded: ${Object.keys(results).filter(key => key !== 'errors').join(', ')}`);
                 }
                 if (results.errors.length > 0) {
-                    this.logger.warn(`Error downloading some${gameMode.name} data from Tarkov Changes: ${results.errors.join(', ')}`);
+                    this.logger.warn(`Error downloading some ${gameMode.name} data: ${results.errors.join(', ')}`);
                 }
                 if (Object.keys(results) === 1) {
-                    this.logger.error(`Error downloading ${gameMode.name} data from Tarkov Changes: ${results.errors.join(', ')}`);
-                    this.discordAlert({
-                        title: `Error(s) updating TC ${gameMode.name} data`,
-                        message: results.errors.join(', '),
-                    })
+                    this.logger.error(`Error(s) downloading ${gameMode.name} data: ${results.errors.join(', ')}`);
+                    results.errors.forEach(errMessage => this.addJobSummary(errMessage, `Error(s) updating TC ${gameMode.name} data`));
                 }
+                return results;
             });
         }
         this.logger.timeEnd('tc-download');

src/tarkov-data-manager/jobs/update-traders.mjs

@@ -140,7 +140,7 @@ class UpdateTradersJob extends DataJob {
 
             if (staleTraderCount > 0) {
                 this.logger.warn(`${staleTraderCount} stale traders`);
-                this.discordAlert({title: 'Stale traders', message: `Running update-traders found ${staleTraderCount} stale ${gameMode.name} traders`});
+                this.addJobSummary(`${staleTraderCount} stale ${gameMode.name} traders`);
             }
 
             let kvName = this.kvName;

src/tarkov-data-manager/jobs/verify-wiki.mjs

@@ -83,12 +83,12 @@ class VerifyWikiJob extends DataJob {
                 }
 
                 if (shouldRemoveCurrentLink && item.wiki_link){
-                    await this.postMessage(item, newWikiLink);
+                    this.addJobSummary(item.name, 'Broken Wiki Link');
                     remoteData.setProperty(item.id, 'wiki_link', '');
                 }
 
                 if (newWikiLink){
-                    await this.postMessage(item, newWikiLink);
+                    this.addJobSummary(item.name, 'Updated Wiki Link');
                     remoteData.setProperty(item.id, 'wiki_link', newWikiLink);
                 }
                 return resolve();
@@ -98,24 +98,6 @@ class VerifyWikiJob extends DataJob {
         // Possibility to POST to a Discord webhook here with cron status details
         this.logger.log(`${missingWikiLinkCount} items still missing a valid wiki link`);
     }
-
-    postMessage = (item, foundNewLink) => {
-        const messageData = {
-            title: 'Broken wiki link',
-            message: item.name
-        };
-
-        if (foundNewLink) {
-            this.logger.succeed(`${item.name} (${item.id}): wiki link updated`);
-
-            messageData.title = 'Updated wiki link';
-            messageData.message = item.name;
-        } else {
-            this.logger.fail(`${item.name} (${item.id}): wiki link removed`);
-        }
-
-        return this.discordAlert(messageData);
-    }
 }
 
 const nameToWikiLink = (name) => {

src/tarkov-data-manager/modules/data-job.mjs

@@ -1,12 +1,14 @@
 import fs from 'node:fs';
 import path from 'node:path';
 
+import  { EmbedBuilder } from 'discord.js';
+
 import cloudflare from './cloudflare.mjs';
 import stellate from './stellate.mjs';
 import TranslationHelper from './translation-helper.mjs';
 import { query, jobComplete, maxQueryRows } from'./db-connection.mjs';
 import JobLogger from './job-logger.mjs';
-import { alert } from './webhook.js';
+import { alert, send as sendWebhook } from './webhook.mjs';
 import webSocketServer from './websocket-server.mjs';
 import tarkovData from'./tarkov-data.mjs';
 import normalizeName from './normalize-name.js';
@@ -78,6 +80,9 @@ class DataJob {
     async start(options) {
         this.startDate = new Date();
         this.kvData = {};
+        this.jobSummary = {
+            general: [],
+        };
         if (this.loadLocales) {
             this.locales = await tarkovData.locales();
             this.translationHelper = new TranslationHelper({
@@ -143,6 +148,38 @@ class DataJob {
                 });
             }
         }
+        if (this.jobSummary.general.length > 0 || Object.keys(this.jobSummary).length > 1) {
+            const embeds = [];
+            for (const messageType in this.jobSummary) {
+                let embed = new EmbedBuilder();
+                embeds.push(embed);
+                if (messageType === 'general') {
+                    embed.setTitle(`${this.name} job`);
+                } else {
+                    embed.setTitle(messageType);
+                }
+                let embedMessage = '';
+                if (this.jobSummary[messageType].length > 0) {
+                    for (let message of this.jobSummary[messageType]) {
+                        if (message.length > 4096) {
+                            message = message.substring(0, 4092)+'...';
+                        }
+                        if (embedMessage.length + message.length > 4096) {
+                            embed.setDescription(embedMessage.trim());
+                            embed = new EmbedBuilder();
+                            embeds.push(embed);
+                            embed.setTitle('(cont)');
+                            embedMessage = message;
+                            continue;
+                        }
+                        embedMessage += '\n' + message;
+                    }
+                    embed.setDescription(embedMessage.trim());
+                }
+            }
+            embeds[embeds.length - 1].setFooter({text: new Date().toLocaleString()});
+            this.discordAlertQueue.push(sendWebhook({embeds}, this.logger));
+        }
         const webhookResults = await Promise.allSettled(this.discordAlertQueue);
         for (const messageResult of webhookResults) {
             if (messageResult.status !== 'rejected') {
@@ -371,6 +408,13 @@ class DataJob {
         };
         return this.jobManager.jobOutput(jobName, options.parentJob, options.gameMode, options.rawOutput);
     }
+
+    addJobSummary = (text, category = 'general') => {
+        if (!this.jobSummary[category]) {
+            this.jobSummary[category] = [];
+        }
+        this.jobSummary[category].push(text);
+    }
 }
 
 export default DataJob;

src/tarkov-data-manager/modules/public-api.mjs

@@ -2,7 +2,7 @@ import { DateTime } from 'luxon';
 
 import { jobOutput } from '../jobs/index.mjs';
 import { query } from './db-connection.mjs';
-import { alert } from './webhook.js';
+import { alert } from './webhook.mjs';
 
 const allMaps = { timestamp: DateTime.now().toFormat('yyyy-LL-dd HH:mm:ss'), maps: [] };
 

src/tarkov-data-manager/modules/tarkov-changes.mjs

@@ -40,6 +40,12 @@ const jsonRequest = async (filename, options) => {
 };
 
 const availableFiles = {
+    achievements: {
+        requestName: 'achievements_list',
+    },
+    achievement_stats: {
+        requestName: 'achievements_statistics',
+    },
     crafts: {},
     credits: {},
     items: {},
@@ -64,7 +70,10 @@ const tarkovChanges = {
         const requestFileName = (availableFiles[file].requestName ?? file) + '.json';
         const saveFileName = file + (gameMode === 'regular' ? '' : `_${gameMode}`) + '.json';
         if (download) {
-            const returnValue = await jsonRequest(requestFileName, options);
+            let returnValue = await jsonRequest(requestFileName, options);
+            if (returnValue.elements) {
+                returnValue = returnValue.elements;
+            }
             fs.writeFileSync(cachePath(saveFileName), JSON.stringify(returnValue, null, 4));
             return returnValue;
         }
@@ -77,6 +86,12 @@ const tarkovChanges = {
             return Promise.reject(error);
         }
     },
+    achievements: (options = defaultOptions) => {
+        return tarkovChanges.get('achievements', merge(options));
+    },
+    achievementStats: (options = defaultOptions) => {
+        return tarkovChanges.get('achievement_stats', merge(options));
+    },
     items: async (options = defaultOptions) => {
         return tarkovChanges.get('items', merge(options));
     },
@@ -105,6 +120,8 @@ const tarkovChanges = {
         options = {...merge(options), download: true};
         const skip = {
             pve: [
+                'achievements',
+                'achievement_stats',
                 'items',
                 'credits',
                 'locale_en',