UBERF-8259: Do not store system model into DB

.vscode/launch.json

@@ -129,7 +129,7 @@
         "MINIO_ACCESS_KEY": "minioadmin",
         "MINIO_SECRET_KEY": "minioadmin",
         "MINIO_ENDPOINT": "localhost",
-        "MODEL_VERSION": "v0.6.286"
+        "MODEL_VERSION": "v0.6.287"
         // "INIT_SCRIPT_URL": "https://raw.githubusercontent.com/hcengineering/init/main/script.yaml",
         // "INIT_WORKSPACE": "onboarding",
       },

dev/tool/src/index.ts

@@ -1503,7 +1503,7 @@ export function devTool (
     .option('-w, --workspace <workspace>', 'Selected workspace only', '')
     .option('-c, --concurrency <concurrency>', 'Number of documents being processed concurrently', '10')
     .action(async (cmd: { workspace: string, concurrency: string }) => {
-      const { mongodbUri, dbUrl } = prepareTools()
+      const { mongodbUri, dbUrl, txes } = prepareTools()
       await withDatabase(mongodbUri, async (db, client) => {
         await withStorage(mongodbUri, async (adapter) => {
           const workspaces = await listWorkspacesPure(db)
@@ -1521,7 +1521,7 @@ export function devTool (
               workspaceUrl: workspace.workspaceUrl ?? ''
             }
 
-            const { pipeline } = await getServerPipeline(toolCtx, mongodbUri, dbUrl, wsUrl)
+            const { pipeline } = await getServerPipeline(toolCtx, txes, mongodbUri, dbUrl, wsUrl)
 
             await migrateMarkup(toolCtx, adapter, wsId, client, pipeline, parseInt(cmd.concurrency))
 

dev/tool/src/markup.ts

@@ -15,7 +15,7 @@ import core, {
 } from '@hcengineering/core'
 import { getMongoClient, getWorkspaceDB } from '@hcengineering/mongo'
 import { type Pipeline, type StorageAdapter } from '@hcengineering/server-core'
-import { connect, fetchModel } from '@hcengineering/server-tool'
+import { connect } from '@hcengineering/server-tool'
 import { jsonToText, markupToYDoc } from '@hcengineering/text'
 import { type Db, type FindCursor, type MongoClient } from 'mongodb'
 
@@ -123,7 +123,7 @@ export async function migrateMarkup (
   pipeline: Pipeline,
   concurrency: number
 ): Promise<void> {
-  const { hierarchy } = await fetchModel(ctx, pipeline)
+  const hierarchy = pipeline.context.hierarchy
 
   const workspaceDb = client.db(workspaceId.name)
 

pods/backup/package.json

@@ -57,6 +57,7 @@
     "dotenv": "~16.0.0",
     "@hcengineering/backup-service": "^0.6.0",
     "@hcengineering/analytics": "^0.6.0",
-    "@hcengineering/analytics-service": "^0.6.0"
+    "@hcengineering/analytics-service": "^0.6.0",
+    "@hcengineering/model-all": "^0.6.0"
   }
 }

pods/backup/src/index.ts

@@ -16,12 +16,19 @@
 import { Analytics } from '@hcengineering/analytics'
 import { configureAnalytics, SplitLogger } from '@hcengineering/analytics-service'
 import { startBackup } from '@hcengineering/backup-service'
-import { MeasureMetricsContext, metricsToString, newMetrics } from '@hcengineering/core'
+import { MeasureMetricsContext, metricsToString, newMetrics, type Tx } from '@hcengineering/core'
 import { type PipelineFactory } from '@hcengineering/server-core'
 import { createBackupPipeline, getConfig } from '@hcengineering/server-pipeline'
 import { writeFile } from 'fs/promises'
 import { join } from 'path'
 
+import builder from '@hcengineering/model-all'
+
+const enabled = (process.env.MODEL_ENABLED ?? '*').split(',').map((it) => it.trim())
+const disabled = (process.env.MODEL_DISABLED ?? '').split(',').map((it) => it.trim())
+
+const model = JSON.parse(JSON.stringify(builder(enabled, disabled).getTxes())) as Tx[]
+
 const metricsContext = new MeasureMetricsContext(
   'backup',
   {},
@@ -58,7 +65,7 @@ const onClose = (): void => {
 startBackup(
   metricsContext,
   (mongoUrl, storageAdapter) => {
-    const factory: PipelineFactory = createBackupPipeline(metricsContext, mongoUrl, {
+    const factory: PipelineFactory = createBackupPipeline(metricsContext, mongoUrl, model, {
       externalStorage: storageAdapter,
       usePassedCtx: true
     })

pods/server/package.json

@@ -72,6 +72,7 @@
     "@hcengineering/server-telegram": "^0.6.0",
     "@hcengineering/pod-telegram-bot": "^0.6.0",
     "@hcengineering/server-ai-bot": "^0.6.0",
-    "@hcengineering/server-ai-bot-resources": "^0.6.0"
+    "@hcengineering/server-ai-bot-resources": "^0.6.0",
+    "@hcengineering/model-all": "^0.6.0"
   }
 }

pods/server/src/server.ts

@@ -14,7 +14,7 @@
 // limitations under the License.
 //
 
-import { type Branding, type BrandingMap, type WorkspaceIdWithUrl } from '@hcengineering/core'
+import { type Branding, type BrandingMap, type Tx, type WorkspaceIdWithUrl } from '@hcengineering/core'
 import { buildStorageFromConfig, getMetricsContext } from '@hcengineering/server'
 
 import { ClientSession, startSessionManager, type ServerFactory, type Session } from '@hcengineering/server'
@@ -25,6 +25,13 @@ import { serverAiBotId } from '@hcengineering/server-ai-bot'
 import { createAIBotAdapter } from '@hcengineering/server-ai-bot-resources'
 import { createServerPipeline, registerServerPlugins, registerStringLoaders } from '@hcengineering/server-pipeline'
 
+import builder from '@hcengineering/model-all'
+
+const enabled = (process.env.MODEL_ENABLED ?? '*').split(',').map((it) => it.trim())
+const disabled = (process.env.MODEL_DISABLED ?? '').split(',').map((it) => it.trim())
+
+const model = JSON.parse(JSON.stringify(builder(enabled, disabled).getTxes())) as Tx[]
+
 registerStringLoaders()
 
 /**
@@ -64,6 +71,7 @@ export function start (
   const pipelineFactory = createServerPipeline(
     metrics,
     dbUrls,
+    model,
     { ...opt, externalStorage, adapterSecurity: rawDbUrl !== undefined },
     {
       serviceAdapters: {

server/middleware/src/model.ts

@@ -14,14 +14,22 @@
 //
 
 import core, {
+  type Doc,
   type LoadModelResponse,
   type MeasureContext,
   type Timestamp,
   type Tx,
+  type TxCUD,
   DOMAIN_TX
 } from '@hcengineering/core'
 import { PlatformError, unknownError } from '@hcengineering/platform'
-import type { Middleware, PipelineContext, TxAdapter, TxMiddlewareResult } from '@hcengineering/server-core'
+import type {
+  Middleware,
+  MiddlewareCreator,
+  PipelineContext,
+  TxAdapter,
+  TxMiddlewareResult
+} from '@hcengineering/server-core'
 import { BaseMiddleware } from '@hcengineering/server-core'
 import crypto from 'node:crypto'
 
@@ -34,28 +42,54 @@ export class ModelMiddleware extends BaseMiddleware implements Middleware {
   lastHashResponse!: Promise<LoadModelResponse>
   model!: Tx[]
 
-  static async create (ctx: MeasureContext, context: PipelineContext, next?: Middleware): Promise<Middleware> {
-    const middleware = new ModelMiddleware(context, next)
+  constructor (
+    context: PipelineContext,
+    next: Middleware | undefined,
+    readonly systemTx: Tx[]
+  ) {
+    super(context, next)
+  }
+
+  static async doCreate (
+    ctx: MeasureContext,
+    context: PipelineContext,
+    next: Middleware | undefined,
+    systemTx: Tx[]
+  ): Promise<Middleware> {
+    const middleware = new ModelMiddleware(context, next, systemTx)
     await middleware.init(ctx)
     return middleware
   }
 
+  static create (tx: Tx[]): MiddlewareCreator {
+    return (ctx, context, next) => {
+      return this.doCreate(ctx, context, next, tx)
+    }
+  }
+
   async init (ctx: MeasureContext): Promise<void> {
     if (this.context.adapterManager == null) {
       throw new PlatformError(unknownError('Adapter manager should be configured'))
     }
     const txAdapter = this.context.adapterManager.getAdapter(DOMAIN_TX, true) as TxAdapter
 
+    const isUserTx = (it: Tx): boolean =>
+      it.modifiedBy !== core.account.System ||
+      (it as TxCUD<Doc>).objectClass === 'contact:class:Person' ||
+      (it as TxCUD<Doc>).objectClass === 'contact:class:PersonAccount'
+
     this.model = await ctx.with('get-model', {}, async (ctx) => {
-      const model = await ctx.with('fetch-model', {}, (ctx) => txAdapter.getModel(ctx))
+      const allUserTxes = await ctx.with('fetch-model', {}, (ctx) => txAdapter.getModel(ctx))
+      const userTxes = allUserTxes.filter((it) => isUserTx(it))
+      const model = this.systemTx.concat(userTxes)
       for (const tx of model) {
         try {
           this.context.hierarchy.tx(tx)
         } catch (err: any) {
           ctx.warn('failed to apply model transaction, skipping', { tx: JSON.stringify(tx), err })
         }
       }
-      this.context.modelDb.addTxes(ctx, model, false)
+      this.context.modelDb.addTxes(ctx, model, true)
       return model
     })
 

server/mongo/src/storage.ts

@@ -1174,7 +1174,9 @@ abstract class MongoAdapterBase implements DbAdapter {
 
   async clean (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]): Promise<void> {
     await ctx.with('clean', {}, async () => {
-      await this.db.collection<Doc>(domain).deleteMany({ _id: { $in: docs } })
+      if (docs.length > 0) {
+        await this.db.collection<Doc>(domain).deleteMany({ _id: { $in: docs } })
+      }
     })
   }
 }
@@ -1693,7 +1695,7 @@ class MongoTxAdapter extends MongoAdapterBase implements TxAdapter {
       )
     }
     model.forEach((tx) => (tx.modifiedBy === core.account.System && !isPersonAccount(tx) ? systemTx : userTx).push(tx))
-    return systemTx.concat(userTx)
+    return this.stripHash(systemTx.concat(userTx)) as Tx[]
   }
 }
 

server/mongo/src/utils.ts

@@ -169,6 +169,7 @@ export class DBCollectionHelper implements DomainHelperOperations {
   }
 
   async init (domain?: Domain): Promise<void> {
+    // Check and create DB if missin
     if (domain === undefined) {
       // Init existing collecfions
       for (const c of (await this.db.listCollections({}, { nameOnly: true }).toArray()).map((it) => it.name)) {

server/server-pipeline/src/pipeline.ts

@@ -10,6 +10,7 @@ import {
   ModelDb,
   type Branding,
   type MeasureContext,
+  type Tx,
   type WorkspaceIdWithUrl
 } from '@hcengineering/core'
 import { createElasticAdapter, createElasticBackupDataAdapter } from '@hcengineering/elastic'
@@ -99,6 +100,7 @@ export function getTxAdapterFactory (
 export function createServerPipeline (
   metrics: MeasureContext,
   dbUrls: string,
+  model: Tx[],
   opt: {
     fullTextUrl: string
     rekoniUrl: string
@@ -137,7 +139,7 @@ export function createServerPipeline (
       DomainFindMiddleware.create,
       DomainTxMiddleware.create,
       DBAdapterInitMiddleware.create,
-      ModelMiddleware.create,
+      ModelMiddleware.create(model),
       DBAdapterMiddleware.create(conf), // Configure DB adapters
       BroadcastMiddleware.create(broadcast)
     ]
@@ -162,6 +164,7 @@ export function createServerPipeline (
 export function createBackupPipeline (
   metrics: MeasureContext,
   dbUrls: string,
+  systemTx: Tx[],
   opt: {
     usePassedCtx?: boolean
     adapterSecurity?: boolean
@@ -206,7 +209,7 @@ export function createBackupPipeline (
       ContextNameMiddleware.create,
       DomainFindMiddleware.create,
       DBAdapterInitMiddleware.create,
-      ModelMiddleware.create,
+      ModelMiddleware.create(systemTx),
       DBAdapterMiddleware.create(conf)
     ]
 
@@ -225,6 +228,7 @@ export function createBackupPipeline (
 
 export async function getServerPipeline (
   ctx: MeasureContext,
+  model: Tx[],
   mongodbUri: string,
   dbUrl: string | undefined,
   wsUrl: WorkspaceIdWithUrl
@@ -240,6 +244,7 @@ export async function getServerPipeline (
   const pipelineFactory = createServerPipeline(
     ctx,
     dbUrls,
+    model,
     {
       externalStorage: storageAdapter,
       fullTextUrl: 'http://localhost:9200',