feat: restore lost markup tool (#6724)

Signed-off-by: Alexander Onnikov <[email protected]>

dev/tool/src/index.ts

@@ -112,7 +112,7 @@ import {
 } from './clean'
 import { changeConfiguration } from './configuration'
 import { moveFromMongoToPG, moveWorkspaceFromMongoToPG } from './db'
-import { fixJsonMarkup, migrateMarkup } from './markup'
+import { fixJsonMarkup, migrateMarkup, restoreLostMarkup } from './markup'
 import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin'
 import { importNotion } from './notion'
 import { fixAccountEmails, renameAccount } from './renameAccount'
@@ -1273,6 +1273,38 @@ export function devTool (
       })
     })
 
+  program.command('show-lost-markup <workspace>').action(async (workspace: string, cmd: any) => {
+    const { mongodbUri } = prepareTools()
+    await withDatabase(mongodbUri, async (db, client) => {
+      await withStorage(mongodbUri, async (adapter) => {
+        try {
+          const workspaceId = getWorkspaceId(workspace)
+          const token = generateToken(systemAccountEmail, workspaceId)
+          const endpoint = await getTransactorEndpoint(token)
+          await restoreLostMarkup(toolCtx, workspaceId, endpoint, adapter, { command: 'show' })
+        } catch (err: any) {
+          console.error(err)
+        }
+      })
+    })
+  })
+
+  program.command('restore-lost-markup <workspace>').action(async (workspace: string, cmd: any) => {
+    const { mongodbUri } = prepareTools()
+    await withDatabase(mongodbUri, async (db, client) => {
+      await withStorage(mongodbUri, async (adapter) => {
+        try {
+          const workspaceId = getWorkspaceId(workspace)
+          const token = generateToken(systemAccountEmail, workspaceId)
+          const endpoint = await getTransactorEndpoint(token)
+          await restoreLostMarkup(toolCtx, workspaceId, endpoint, adapter, { command: 'restore' })
+        } catch (err: any) {
+          console.error(err)
+        }
+      })
+    })
+  })
+
   program.command('fix-bw-workspace <workspace>').action(async (workspace: string) => {
     const { mongodbUri } = prepareTools()
     await withStorage(mongodbUri, async (adapter) => {

dev/tool/src/markup.ts

@@ -3,20 +3,28 @@ import core, {
   type AnyAttribute,
   type Class,
   type Client as CoreClient,
+  type CollaborativeDoc,
   type Doc,
+  type DocIndexState,
   type Domain,
   type Hierarchy,
+  type Markup,
   type MeasureContext,
   type Ref,
+  type TxMixin,
+  type TxCreateDoc,
+  type TxUpdateDoc,
   type WorkspaceId,
   RateLimiter,
   collaborativeDocParse,
-  makeCollaborativeDoc
+  makeCollaborativeDoc,
+  SortingOrder,
+  TxProcessor
 } from '@hcengineering/core'
 import { getMongoClient, getWorkspaceDB } from '@hcengineering/mongo'
 import { type Pipeline, type StorageAdapter } from '@hcengineering/server-core'
 import { connect } from '@hcengineering/server-tool'
-import { jsonToText, markupToYDoc } from '@hcengineering/text'
+import { isEmptyMarkup, jsonToText, markupToYDoc } from '@hcengineering/text'
 import { type Db, type FindCursor, type MongoClient } from 'mongodb'
 
 export async function fixJsonMarkup (
@@ -212,3 +220,124 @@ async function processMigrateMarkupFor (
 
   console.log('processed', processed)
 }
+
+export async function restoreLostMarkup (
+  ctx: MeasureContext,
+  workspaceId: WorkspaceId,
+  transactorUrl: string,
+  storageAdapter: StorageAdapter,
+  { command }: { command: 'show' | 'restore' }
+): Promise<void> {
+  const connection = (await connect(transactorUrl, workspaceId, undefined, {
+    mode: 'backup'
+  })) as unknown as CoreClient
+
+  try {
+    const hierarchy = connection.getHierarchy()
+    const classes = hierarchy.getDescendants(core.class.Doc)
+
+    for (const _class of classes) {
+      const isAttachedDoc = hierarchy.isDerived(_class, core.class.AttachedDoc)
+
+      const attributes = hierarchy.getAllAttributes(_class)
+      const attrs = Array.from(attributes.values()).filter((p) => p.type._class === core.class.TypeCollaborativeDoc)
+
+      // ignore classes with no collaborative attributes
+      if (attrs.length === 0) continue
+
+      const docs = await connection.findAll(_class, { _class })
+      for (const doc of docs) {
+        for (const attr of attrs) {
+          const value = hierarchy.isMixin(attr.attributeOf)
+            ? ((doc as any)[attr.attributeOf]?.[attr.name] as CollaborativeDoc)
+            : ((doc as any)[attr.name] as CollaborativeDoc)
+
+          if (value == null || value === '') continue
+
+          const { documentId } = collaborativeDocParse(value)
+          const stat = await storageAdapter.stat(ctx, workspaceId, documentId)
+          if (stat !== undefined) continue
+
+          const query = isAttachedDoc
+            ? {
+                'tx.objectId': doc._id,
+                'tx._class': { $in: [core.class.TxCreateDoc, core.class.TxUpdateDoc] }
+              }
+            : {
+                objectId: doc._id
+              }
+
+          let restored = false
+
+          // try to restore by txes
+          // we need last tx that modified the attribute
+
+          const txes = await connection.findAll(isAttachedDoc ? core.class.TxCollectionCUD : core.class.TxCUD, query, {
+            sort: { modifiedOn: SortingOrder.Descending }
+          })
+          for (const tx of txes) {
+            const innerTx = TxProcessor.extractTx(tx)
+
+            let markup: string | undefined
+            if (innerTx._class === core.class.TxMixin) {
+              const mixinTx = innerTx as TxMixin<Doc, Doc>
+              markup = (mixinTx.attributes as any)[attr.name]
+            } else if (innerTx._class === core.class.TxCreateDoc) {
+              const createTx = innerTx as TxCreateDoc<Doc>
+              markup = (createTx.attributes as any)[attr.name]
+            } else if (innerTx._class === core.class.TxUpdateDoc) {
+              const updateTex = innerTx as TxUpdateDoc<Doc>
+              markup = (updateTex.operations as any)[attr.name]
+            } else {
+              continue
+            }
+
+            if (markup === undefined || !markup.startsWith('{')) continue
+            if (isEmptyMarkup(markup)) continue
+
+            console.log(doc._class, doc._id, attr.name, markup)
+            if (command === 'restore') {
+              const ydoc = markupToYDoc(markup, attr.name)
+              await saveCollaborativeDoc(storageAdapter, workspaceId, value, ydoc, ctx)
+            }
+            restored = true
+            break
+          }
+
+          if (restored) continue
+
+          // try to restore by doc index state
+          const docIndexState = await connection.findOne(core.class.DocIndexState, {
+            _id: doc._id as Ref<DocIndexState>
+          })
+          if (docIndexState !== undefined) {
+            // document:class:Document%content#content#base64
+            const attrName = `${doc._class}%${attr.name}#content#base64`
+            const base64: string | undefined = docIndexState.attributes[attrName]
+            if (base64 !== undefined) {
+              const text = Buffer.from(base64, 'base64').toString()
+              if (text !== '') {
+                const markup: Markup = JSON.stringify({
+                  type: 'doc',
+                  content: [
+                    {
+                      type: 'paragraph',
+                      content: [{ type: 'text', text, marks: [] }]
+                    }
+                  ]
+                })
+                console.log(doc._class, doc._id, attr.name, markup)
+                if (command === 'restore') {
+                  const ydoc = markupToYDoc(markup, attr.name)
+                  await saveCollaborativeDoc(storageAdapter, workspaceId, value, ydoc, ctx)
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  } finally {
+    await connection.close()
+  }
+}