From aa8fe53432373f7b76b19d1e817f4fde07b02471 Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Sun, 7 Apr 2024 14:23:24 -0500 Subject: [PATCH 1/4] Add removeDeadChapters script --- package.json | 1 + src/controllers/mediaRef.ts | 22 +++++++++++++++++ src/scripts/mediaRefs/removeDeadChapters.ts | 26 +++++++++++++++++++++ 3 files changed, 49 insertions(+) create mode 100644 src/scripts/mediaRefs/removeDeadChapters.ts diff --git a/package.json b/package.json index f9d8de29..29a5d65f 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "dev:scripts:receiveErrorMessageFromQueue": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/queue/receiveErrorMessageFromQueue.ts", "dev:scripts:refreshEpisodesMostRecentMaterializedView": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/episodes/refreshEpisodesMostRecentMaterializedView.ts", "dev:scripts:refreshMediaRefsVideosMaterializedView": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/mediaRefs_videos/refreshMediaRefsVideosMaterializedView.ts", + "dev:scripts:removeDeadChapters": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/mediaRefs/removeDeadChapters.ts", "dev:scripts:removeDeadEpisodes": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/episodes/removeDeadEpisodes.ts", "dev:scripts:syncWithFeedUrlsCSVDump": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/podcastIndex/syncWithFeedUrlsCSVDump.ts", "dev:scripts:updateRecentEpisodesTables": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/recentEpisodes/updateRecentEpisodesTables.ts", diff --git a/src/controllers/mediaRef.ts b/src/controllers/mediaRef.ts index 48b8857b..7109af32 100644 --- a/src/controllers/mediaRef.ts +++ b/src/controllers/mediaRef.ts @@ -365,6 +365,27 @@ const refreshMediaRefsVideosMaterializedView = async () => { await em.query('REFRESH MATERIALIZED VIEW CONCURRENTLY "mediaRefs_videos"') } +const removeDeadChapters = async () => { + const em = await getConnection().createEntityManager() + try { + for (let i = 0; i < 10000; i++) { + console.log('removeDeadChapters i:', i) + await em.query(` + DELETE FROM "mediaRefs" + WHERE "id" IN ( + SELECT "id" + FROM "mediaRefs" + WHERE "isOfficialChapter" = TRUE + AND "isPublic" = FALSE + LIMIT 100 + ); + `) + } + } catch (error) { + console.log('deleteHiddenMediaRefs error:', error) + } +} + export { createMediaRef, deleteMediaRef, @@ -374,6 +395,7 @@ export { getPublicMediaRefsByEpisodeMediaUrl, getPublicMediaRefsByEpisodeGuid, refreshMediaRefsVideosMaterializedView, + removeDeadChapters, updateMediaRef, updateSoundBites } diff --git a/src/scripts/mediaRefs/removeDeadChapters.ts b/src/scripts/mediaRefs/removeDeadChapters.ts new file mode 100644 index 00000000..9890a22b --- /dev/null +++ b/src/scripts/mediaRefs/removeDeadChapters.ts @@ -0,0 +1,26 @@ +import { removeDeadChapters } from '~/controllers/mediaRef' +import { connectToDb } from '~/lib/db' +;(async function () { + try { + const customLimit = (process.argv.length > 2 ? process.argv[2] : 100000) as any + const customLimitInt = parseInt(customLimit, 10) + + const customOffset = (process.argv.length > 3 ? process.argv[3] : 0) as any + const customOffsetInt = parseInt(customOffset, 10) + + if (!customLimitInt) { + console.log('customLimit parameter must be an integer.') + return + } + + if (!customOffsetInt && customOffsetInt !== 0) { + console.log('customOffset parameter must be an integer.') + return + } + await connectToDb() + await removeDeadChapters() + } catch (error) { + console.log(error) + } + return +})() From 4cf54bda8bb24872f849c17cf5aff21fcedf19b8 Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Sun, 7 Apr 2024 14:23:53 -0500 Subject: [PATCH 2/4] Convert spaces to %20 in chapter urls --- src/controllers/episode.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/controllers/episode.ts b/src/controllers/episode.ts index 583d4a16..d6fbf6f2 100644 --- a/src/controllers/episode.ts +++ b/src/controllers/episode.ts @@ -776,6 +776,9 @@ const retrieveLatestChapters = async (id, includeNonToc = true) => { const chapterHash = uuidv5(JSON.stringify(chapterInfoToHash), uuidNIL) + const imageUrl = newChapter?.img?.replace(/ /g, '%20') || null + const linkUrl = newChapter?.url?.replace(/ /g, '%20') || null + // If a chapter with that chapterHash already exists, then update it. // If it does not exist, then create a new mediaRef with isOfficialChapter = true. const existingChapter = existingChapters.find((x) => x.chapterHash === chapterHash) @@ -784,10 +787,10 @@ const retrieveLatestChapters = async (id, includeNonToc = true) => { { endTime: roundedEndTime, id: existingChapter.id, - imageUrl: newChapter.img || null, + imageUrl, isOfficialChapter: true, isPublic: true, - linkUrl: newChapter.url || null, + linkUrl, startTime: roundedStartTime, title: newChapter.title, episodeId: id, @@ -799,10 +802,10 @@ const retrieveLatestChapters = async (id, includeNonToc = true) => { } else { await createMediaRef({ endTime: roundedEndTime, - imageUrl: newChapter.img || null, + imageUrl, isOfficialChapter: true, isPublic: true, - linkUrl: newChapter.url || null, + linkUrl, startTime: roundedStartTime, title: newChapter.title, owner: superUserId, From 9d0e33473e02df48aa1911aa41b6b625f9c31c8f Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Wed, 10 Apr 2024 14:48:49 -0500 Subject: [PATCH 3/4] Validate podcastGuid is uuid before saving --- src/services/parser.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/services/parser.ts b/src/services/parser.ts index 36bddcb0..0b19ce8d 100644 --- a/src/services/parser.ts +++ b/src/services/parser.ts @@ -11,6 +11,7 @@ import { podcastItunesTypeDefaultValue } from 'podverse-shared' import { getRepository, In, Not } from 'typeorm' +import isUUID from 'validator/lib/isUUID' import { config } from '~/config' import { updateSoundBites } from '~/controllers/mediaRef' import { getPodcast } from '~/controllers/podcast' @@ -371,7 +372,10 @@ export const parseFeedUrl = async (feedUrl, forceReparsing = false, cacheBust = // guid is deprecated podcast.guid = meta.guid // podcastGuid is the column we want to use going forward - podcast.podcastGuid = meta.guid + + if (meta.guid && isUUID(meta.guid)) { + podcast.podcastGuid = meta.guid + } const hasNewImageUrl = meta.imageURL && podcast.imageUrl !== meta.imageURL podcast.imageUrl = meta.imageURL From 7ca14a57f4ea560bdc3213be9c59925b49876c7a Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Wed, 10 Apr 2024 14:49:05 -0500 Subject: [PATCH 4/4] Bump to version 4.16.5 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 29a5d65f..d4c276a0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "podverse-api", - "version": "4.16.4", + "version": "4.16.5", "description": "Data API, database migration scripts, and backend services for all Podverse models.", "contributors": [ "Mitch Downey"