Improve migrations

* Schedule items are inserted with batchInsert
  cuts migration time to one-tenth
* add tono_encumbered transitional flag to video
   we can migrate to a better
   schema later
* add active to organization
* experiment with Objection
   as query builder (seems to beat NIH!)

cli/src/migrate-fkweb/migrateCategories.ts

@@ -0,0 +1,61 @@
+import { fkweb } from "./fkwebDatabase"
+import { db } from "../db"
+import { FkUser, RoleUserMap, Users } from "../tableTypes"
+
+// TODO: Add date of birth, phone number, identity confirmation
+
+export const migrateUsers = async () => {
+  const query = await fkweb<FkUser>("fk_user").select(
+    "id",
+    "password",
+    "last_login",
+    "is_superuser",
+    "first_name",
+    "last_name",
+    "email",
+    "is_active",
+    "date_joined",
+    "date_of_birth",
+    "identity_confirmed",
+    "phone_number",
+  )
+
+  await db<Users>("users").insert({
+    id: 0,
+    password: "",
+    email: "[email protected]",
+    name: "RESERVED USER FOR MIGRATIONS",
+  })
+
+  await Promise.all(
+    query.map(
+      async ({
+        id,
+        password,
+        last_login,
+        is_superuser,
+        first_name,
+        last_name,
+        email,
+        is_active,
+        date_joined,
+        date_of_birth,
+        identity_confirmed,
+        phone_number,
+      }) => {
+        await db<Users>("users").insert({
+          id,
+          password,
+          banned: !is_active,
+          email,
+          last_logged_in_at: last_login,
+          name: `${first_name} ${last_name}`,
+          created_at: date_joined,
+        })
+        if (is_superuser)
+          await db<RoleUserMap>("role_user_map").insert({ user_id: id, role_id: 1 })
+      },
+    ),
+  )
+  await db.raw("SELECT setval('users_id_seq', (SELECT MAX(id) FROM users));")
+}

cli/src/migrate-fkweb/migrateOrganizations.ts

@@ -40,6 +40,7 @@ export const migrateOrganizations = async () => {
             homepage,
             id,
             name,
+            active: fkmember,
             postal_address: postal_address || "",
             street_address: street_address || "",
           })

cli/src/migrate-fkweb/migrateSchedule.ts

@@ -24,27 +24,32 @@ export const migrateSchedule = async () => {
   log.info(`${scheduleItems.length} items read.`)
 
   log.info(`Building map of known videos`)
-  const knownVideos = (await db<Videos>("videos").select("id")).map(({ id }) => id)
+  const knownVideos = new Set(
+    (await db<Videos>("videos").select("id")).map(({ id }) => id),
+  )
 
-  log.info(`Building and storing schedule`)
-  await Promise.all(
-    scheduleItems.map(async ({ id, video_id, starttime }) => {
+  log.info(`Building schedule`)
+  const entries = scheduleItems
+    .filter(({ video_id }) => {
       if (!video_id) {
         skippedBecauseVideoNull += 1
-        return
+        return false
       }
-      if (!knownVideos.find((id) => id === video_id)) {
+      const known = knownVideos.has(video_id)
+      if (!known) {
         skippedBecauseVideoDoesntExist += 1
-        return
+        return false
       }
+      return true
+    })
+    .map(({ id, video_id, starttime: starts_at }) => ({
+      id,
+      video_id: video_id!,
+      starts_at,
+    }))
 
-      return db<JukeboxEntries>("jukebox_entries").insert({
-        id,
-        video_id,
-        starts_at: starttime,
-      })
-    }),
-  )
+  log.info(`Inserting ${entries.length} schedule items`)
+  await db.batchInsert("jukebox_entries", entries)
   log.warn(
     `${skippedBecauseVideoNull} schedule items were skipped because video_id was null`,
   )

cli/src/migrate-fkweb/migrateVideos.ts

@@ -1,9 +1,42 @@
 import { fkweb } from "./fkwebDatabase"
 import { db } from "../db"
-import { FkVideo, Organizations, Videos } from "../tableTypes"
+import { Categories, FkVideo, Organizations, Videos } from "../tableTypes"
 import { getOriginalIds } from "./migrateVideoFiles"
 import { log } from "./log"
 
+// Uses the "name" field of both tables to return the new category ID
+const mapOldCategoryIdToNew = async (oldId: number) => {
+  // Get name for a given ID from old database:
+  const oldName = await fkweb<Categories>("fk_category")
+    .select("name")
+    .where("id", oldId)
+    .first()
+
+  if (!oldName) throw new Error(`Category ${oldId} not found in old database`)
+
+  const result = await db<Categories>("categories")
+    .select("id")
+    .where("name", oldName.name)
+    .first()
+
+  if (!result)
+    throw new Error(`Category ${oldId}: ${oldName.name} not found in new database`)
+  return result.id
+}
+
+const migrateVideoCategories = async (videoId: number) => {
+  const categoriesForVideo = await fkweb("fk_video_categories").where("video_id", videoId)
+
+  await Promise.all(
+    categoriesForVideo.map(async ({ category_id }) => {
+      await db("video_category_map").insert({
+        video_id: videoId,
+        category_id: await mapOldCategoryIdToNew(category_id),
+      })
+    }),
+  )
+}
+
 // TODO: Fix editorId and fkmember, resolve non-unique brregId
 // TODO: Resolve description issue, discontinuity between SQL schema and GraphQL schema
 //    Graph returns Cannot return null for non-nullable field Video.description.
@@ -60,7 +93,7 @@ export const migrateVideos = async () => {
         }
 
         try {
-          return await db<Videos>("videos").insert({
+          const video = await db<Videos>("videos").insert({
             id,
             title: name,
             description: header || "",
@@ -70,7 +103,12 @@ export const migrateVideos = async () => {
             updated_at: updated_time || new Date(),
             media_id: originals[id],
             published: publish_on_web,
+            tono_encumbered: has_tono_records,
           })
+
+          await migrateVideoCategories(id)
+
+          return video
         } catch (e) {
           log.warn(`Skipping video ${id} due to database error`)
           console.log(e)

cli/src/tableTypes.ts

@@ -299,6 +299,7 @@ export type Organizations = {
   homepage: string | null
   postal_address: string
   street_address: string
+  active: boolean
   editor_id: number
   created_at: Date
   updated_at: Date
@@ -384,4 +385,5 @@ export type Videos = {
   first_name: string
   last_name: string
   published: boolean
+  tono_encumbered: boolean
 }

migrations/20211217184723_categories.js

@@ -2,57 +2,57 @@
 
 const starterCategories = [
   {
-    key: "society",
+    slug: "society",
     name: "Samfunn og politikk",
     description: "",
   },
   {
-    key: "solidarity",
+    slug: "solidarity",
     name: "Solidaritet og bistand",
     description: "",
   },
   {
-    key: "religion",
+    slug: "religion",
     name: "Religion og livssyn",
     description: "",
   },
   {
-    key: "minorities",
+    slug: "minorities",
     name: "Minoriteter",
     description: "",
   },
   {
-    key: "welfare",
+    slug: "welfare",
     name: "Velferd",
     description: "",
   },
   {
-    key: "culture",
+    slug: "culture",
     name: "Kultur",
     description: "",
   },
   {
-    key: "sports",
+    slug: "sports",
     name: "Idrett",
     description: "",
   },
   {
-    key: "public-security",
+    slug: "public-security",
     name: "Beredskap",
     description: "",
   },
   {
-    key: "children",
+    slug: "children",
     name: "Barn og ungdom",
     description: "",
   },
   {
-    key: "technology",
+    slug: "technology",
     name: "Teknologi",
     description: "",
   },
   {
-    key: "other",
+    slug: "other",
     name: "Annet",
     description: "",
   },
@@ -65,7 +65,7 @@ exports.up = async (knex) => {
   await knex.schema.createTable("categories", (table) => {
     table.increments("id")
 
-    table.string("key").notNullable().unique()
+    table.string("slug").notNullable().unique()
     table.string("name").notNullable().unique()
     table.text("description").notNullable()
   })

migrations/20230727124759_add_membership_status.js

@@ -0,0 +1,21 @@
+//@ts-check
+
+/**
+ * @param {import("knex").Knex} knex
+ */
+exports.up = async (knex) => {
+  // Alter the "organizations" table to add an "active" column
+  await knex.schema.alterTable("organizations", (table) => {
+    table.boolean("active").defaultTo(false)
+  })
+}
+
+/**
+ * @param {import("knex").Knex} knex
+ */
+exports.down = async (knex) => {
+  // Remove the "active" column
+  await knex.schema.alterTable("organizations", (table) => {
+    table.dropColumn("active")
+  })
+}

migrations/20230727151557_video_tono_field.js

@@ -0,0 +1,21 @@
+//@ts-check
+
+// Add a tonoEncumbered field to videos to indicate copyright encumberance
+
+/**
+ * @param {import("knex").Knex} knex
+ */
+exports.up = async (knex) => {
+  await knex.schema.table("videos", (table) => {
+    table.boolean("tono_encumbered").notNullable().defaultTo(false)
+  })
+}
+
+/**
+ * @param {import("knex").Knex} knex
+ */
+exports.down = async (knex) => {
+  await knex.schema.table("videos", (table) => {
+    table.dropColumn("tono_encumbered")
+  })
+}

package.json

@@ -35,6 +35,7 @@
         "koa-session": "^6.4.0",
         "koa2-swagger-ui": "^5.8.0",
         "libxmljs2": "^0.32.0",
+        "objection": "^3.1.1",
         "p-limit": "^4.0.0",
         "pg": "^8.11.0",
         "swagger-jsdoc": "^6.2.8",

src/modules/auth/middleware/requireSecretKey.ts

@@ -1,9 +1,14 @@
 import { Middleware } from "koa"
 import { SECRET_KEY, SECRET_KEY_HEADER } from "../constants"
+import { IS_DEV } from "../../core/constants"
+import { log } from "../../core/log"
 
 export const requireSecretKey = (): Middleware => (context, next) => {
   if (context.get(SECRET_KEY_HEADER) !== SECRET_KEY)
-    context.throw(400, `${SECRET_KEY_HEADER} missing or incorrect`)
+    if (IS_DEV) {
+      log.debug(`Permitting request without secret key in dev mode`)
+      return next()
+    } else context.throw(400, `${SECRET_KEY_HEADER} missing or incorrect`)
 
   return next()
 }

src/modules/category/helpers/serializeCategory.ts

@@ -9,7 +9,7 @@ import { CategoryData } from "../models/categoryModel"
  *       properties:
  *         id:
  *           type: number
- *         key:
+ *         slug:
  *           type: string
  *         name:
  *           type: string

src/modules/category/models/categoryModel.ts

@@ -3,14 +3,14 @@ import { Model } from "../../db/classes/Model"
 export type CategoryData = {
   id: number
 
-  key: string
+  slug: string
   name: string
   description: string
 }
 
 export const categoryModel = new Model<CategoryData>({
   tableName: "categories",
-  columns: ["id", "key", "name", "description"],
+  columns: ["id", "slug", "name", "description"],
   structure: {
     prefix: "category",
     property: "category",