feat: impl dal forkDb (#202)

<!--
Thank you for your pull request. Please review below requirements.
Bug fixes and new features should include tests and possibly benchmarks.
Contributors guide:
https://github.com/eggjs/egg/blob/master/CONTRIBUTING.md

感谢您贡献代码。请确认下列 checklist 的完成情况。
Bug 修复和新功能必须包含测试，必要时请附上性能测试。
Contributors guide:
https://github.com/eggjs/egg/blob/master/CONTRIBUTING.md
-->

##### Checklist
<!-- Remove items that do not apply. For completed items, change [ ] to
[x]. -->

- [ ] `npm test` passes
- [ ] tests and/or benchmarks are included
- [ ] documentation is changed or added
- [ ] commit message follows commit guidelines

##### Affected core subsystem(s)
<!-- Provide affected core subsystem(s). -->


##### Description of change
<!-- Provide a description of the change below this comment. -->

<!--
- any feature?
- close https://github.com/eggjs/egg/ISSUE_URL
-->

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

- **New Features**
- Introduced `DatabaseForker` for efficient database forking,
particularly useful in unit testing environments.
- Added configuration options for database forking in various modules
and tests.
- Enhanced `MysqlDataSource` with new properties for improved database
connection handling.

- **Documentation**
- Updated README with instructions for unit test database configuration.

- **Tests**
- Adjusted database initialization and cleanup processes in unit tests
to utilize `DatabaseForker`.
- Modified database configurations in test fixtures to support new
forking functionality.

- **Refactor**
- Refactored database connection and forking logic across multiple
modules to integrate `DatabaseForker`.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

core/dal-runtime/index.ts

@@ -6,3 +6,4 @@ export * from './src/SqlMapLoader';
 export * from './src/DataSource';
 export * from './src/MySqlDataSource';
 export * from './src/TableModelInstanceBuilder';
+export * from './src/DatabaseForker';

core/dal-runtime/src/DatabaseForker.ts

@@ -0,0 +1,68 @@
+import { DataSourceOptions } from './MySqlDataSource';
+import { RDSClient } from '@eggjs/rds';
+import path from 'node:path';
+import fs from 'node:fs/promises';
+import assert from 'node:assert';
+
+export class DatabaseForker {
+  private readonly env: string;
+  private readonly options: DataSourceOptions;
+
+  constructor(env: string, options: DataSourceOptions) {
+    this.env = env;
+    this.options = options;
+  }
+
+  shouldFork() {
+    return this.env === 'unittest' && this.options.forkDb;
+  }
+
+  async forkDb(dalDir: string) {
+    assert(this.shouldFork(), 'fork db only run in unittest');
+    // 尽早判断不应该 fork，避免对 rds pool 配置造成污染
+    try {
+      await fs.access(dalDir);
+    } catch (_) {
+      return;
+    }
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const { name, initSql, forkDb, database, ...mysqlOptions } = this.options;
+    const client = new RDSClient(Object.assign(mysqlOptions));
+    const conn = await client.getConnection();
+    await this.doCreateUtDb(conn);
+    await this.forkTables(conn, dalDir);
+    conn.release();
+    await client.end();
+  }
+
+  private async forkTables(conn, dalDir: string) {
+    const sqlDir = path.join(dalDir, 'structure');
+    const structureFiles = await fs.readdir(sqlDir);
+    const sqlFiles = structureFiles.filter(t => t.endsWith('.sql'));
+    for (const sqlFile of sqlFiles) {
+      await this.doForkTable(conn, path.join(sqlDir, sqlFile));
+    }
+  }
+
+  private async doForkTable(conn, sqlFileName: string) {
+    const sqlFile = await fs.readFile(sqlFileName, 'utf8');
+    const sqls = sqlFile.split(';').filter(t => !!t.trim());
+    for (const sql of sqls) {
+      await conn.query(sql);
+    }
+  }
+
+  private async doCreateUtDb(conn) {
+    await conn.query(`CREATE DATABASE IF NOT EXISTS ${this.options.database};`);
+    await conn.query(`use ${this.options.database};`);
+  }
+
+  async destroy() {
+    assert(this.shouldFork(), 'fork db only run in unittest');
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const { name, initSql, forkDb, database, ...mysqlOptions } = this.options;
+    const client = new RDSClient(Object.assign(mysqlOptions));
+    await client.query(`DROP DATABASE ${database}`);
+    await client.end();
+  }
+}

core/dal-runtime/src/MySqlDataSource.ts

@@ -1,12 +1,12 @@
 import { RDSClient } from '@eggjs/rds';
-// TODO fix export
-import type { RDSClientOptions } from '@eggjs/rds/lib/types';
+import type { RDSClientOptions } from '@eggjs/rds';
 import Base from 'sdk-base';
 
 export interface DataSourceOptions extends RDSClientOptions {
   name: string;
   // default is select 1 + 1;
   initSql?: string;
+  forkDb?: boolean;
 }
 
 const DEFAULT_OPTIONS: RDSClientOptions = {
@@ -16,18 +16,22 @@ const DEFAULT_OPTIONS: RDSClientOptions = {
 };
 
 export class MysqlDataSource extends Base {
-  private readonly client: RDSClient;
+  private client: RDSClient;
   private readonly initSql: string;
   readonly name: string;
   readonly timezone?: string;
+  readonly rdsOptions: RDSClientOptions;
+  readonly forkDb?: boolean;
 
   constructor(options: DataSourceOptions) {
     super({ initMethod: '_init' });
-    const { name, initSql, ...mysqlOptions } = options;
-    this.client = new RDSClient(Object.assign({}, DEFAULT_OPTIONS, mysqlOptions));
+    const { name, initSql, forkDb, ...mysqlOptions } = options;
+    this.forkDb = forkDb;
     this.initSql = initSql ?? 'SELECT 1 + 1';
     this.name = name;
     this.timezone = options.timezone;
+    this.rdsOptions = Object.assign({}, DEFAULT_OPTIONS, mysqlOptions);
+    this.client = new RDSClient(this.rdsOptions);
   }
 
   protected async _init() {

core/dal-runtime/test/DAO.test.ts

@@ -5,37 +5,40 @@ import { Foo } from './fixtures/modules/dal/Foo';
 import { TableModel } from '@eggjs/dal-decorator';
 import path from 'node:path';
 import { DataSource } from '../src/DataSource';
-import { SqlGenerator } from '../src/SqlGenerator';
 import FooDAO from './fixtures/modules/dal/dal/dao/FooDAO';
+import { DatabaseForker } from '../src/DatabaseForker';
 
 describe('test/DAO.test.ts', () => {
   let dataSource: DataSource<Foo>;
   let tableModel: TableModel<Foo>;
+  let forker: DatabaseForker;
 
   before(async () => {
-    const mysql = new MysqlDataSource({
+    const mysqlOptions = {
       name: 'foo',
       host: '127.0.0.1',
       user: 'root',
-      database: 'test',
+      database: 'test_runtime',
       timezone: '+08:00',
       initSql: 'SET GLOBAL time_zone = \'+08:00\';',
-    });
+      forkDb: true,
+    };
+    forker = new DatabaseForker('unittest', mysqlOptions);
+    await forker.forkDb(path.join(__dirname, './fixtures/modules/dal/dal'));
+
+    const mysql = new MysqlDataSource(mysqlOptions);
     await mysql.ready();
-    await mysql.query('DROP TABLE IF EXISTS egg_foo');
 
     tableModel = TableModel.build(Foo);
-
-    const sqlGenerator = new SqlGenerator();
-    const createTableSql = sqlGenerator.generate(tableModel);
-
-    await mysql.query(createTableSql);
-
     const sqlMapLoader = new SqlMapLoader(tableModel, path.join(__dirname, './fixtures/modules/dal'), console as any);
     const sqlMap = sqlMapLoader.load();
     dataSource = new DataSource(tableModel, mysql, sqlMap);
   });
 
+  after(async () => {
+    await forker.destroy();
+  });
+
   it('execute should work', async () => {
     const foo = new Foo();
     foo.name = 'name';

core/dal-runtime/test/DataSource.test.ts

@@ -7,36 +7,38 @@ import path from 'node:path';
 import { DataSource } from '../src/DataSource';
 import { TableModelInstanceBuilder } from '../src/TableModelInstanceBuilder';
 import { DeleteResult, InsertResult, UpdateResult } from '@eggjs/rds/lib/types';
-import { SqlGenerator } from '../src/SqlGenerator';
+import { DatabaseForker } from '../src/DatabaseForker';
 
 describe('test/Datasource.test.ts', () => {
   let dataSource: DataSource<Foo>;
   let tableModel: TableModel<Foo>;
+  let forker: DatabaseForker;
 
   before(async () => {
-    const mysql = new MysqlDataSource({
+    const mysqlOptions = {
       name: 'foo',
       host: '127.0.0.1',
       user: 'root',
-      database: 'test',
+      database: 'test_runtime',
       timezone: '+08:00',
       initSql: 'SET GLOBAL time_zone = \'+08:00\';',
-    });
+      forkDb: true,
+    };
+    forker = new DatabaseForker('unittest', mysqlOptions);
+    await forker.forkDb(path.join(__dirname, './fixtures/modules/dal/dal'));
+    const mysql = new MysqlDataSource(mysqlOptions);
     await mysql.ready();
-    await mysql.query('DROP TABLE IF EXISTS egg_foo');
 
     tableModel = TableModel.build(Foo);
-
-    const sqlGenerator = new SqlGenerator();
-    const createTableSql = sqlGenerator.generate(tableModel);
-
-    await mysql.query(createTableSql);
-
     const sqlMapLoader = new SqlMapLoader(tableModel, path.join(__dirname, './fixtures/modules/dal'), console as any);
     const sqlMap = sqlMapLoader.load();
     dataSource = new DataSource(tableModel, mysql, sqlMap);
   });
 
+  after(async () => {
+    await forker.destroy();
+  });
+
   it('execute should work', async () => {
     const foo = new Foo();
     foo.name = 'name';

plugin/dal/README.md

@@ -637,3 +637,15 @@ dataSource:
 ```sql
 SELECT @@GLOBAL.time_zone;
 ```
+
+## Unittest
+可以在 `module.yml` 中开启 forkDb 配置，即可实现 unittest 环境自动创建数据库
+
+```yaml
+# module.yml
+dataSource:
+  foo:
+    # 开启 ci 环境自动创建数据库
+    forkDb: true
+
+```

plugin/dal/app.ts

@@ -17,7 +17,7 @@ export default class ControllerAppBootHook {
   }
 
   configWillLoad() {
-    this.dalModuleLoadUnitHook = new DalModuleLoadUnitHook(this.app.moduleConfigs);
+    this.dalModuleLoadUnitHook = new DalModuleLoadUnitHook(this.app.config.env, this.app.moduleConfigs);
     this.dalTableEggPrototypeHook = new DalTableEggPrototypeHook(this.app.logger);
     this.app.eggPrototypeLifecycleUtil.registerLifecycle(this.dalTableEggPrototypeHook);
     this.app.loadUnitLifecycleUtil.registerLifecycle(this.dalModuleLoadUnitHook);

plugin/dal/lib/DalModuleLoadUnitHook.ts

@@ -1,13 +1,16 @@
 import { MysqlDataSourceManager } from './MysqlDataSourceManager';
+import path from 'node:path';
 import { LifecycleHook } from '@eggjs/tegg-lifecycle';
 import { ModuleConfigHolder } from '@eggjs/tegg-common-util';
-import { DataSourceOptions } from '@eggjs/dal-runtime';
+import { DatabaseForker, DataSourceOptions } from '@eggjs/dal-runtime';
 import { LoadUnit, LoadUnitLifecycleContext } from '@eggjs/tegg/helper';
 
 export class DalModuleLoadUnitHook implements LifecycleHook<LoadUnitLifecycleContext, LoadUnit> {
   private readonly moduleConfigs: Record<string, ModuleConfigHolder>;
+  private readonly env: string;
 
-  constructor(moduleConfigs: Record<string, ModuleConfigHolder>) {
+  constructor(env: string, moduleConfigs: Record<string, ModuleConfigHolder>) {
+    this.env = env;
     this.moduleConfigs = moduleConfigs;
   }
 
@@ -17,11 +20,17 @@ export class DalModuleLoadUnitHook implements LifecycleHook<LoadUnitLifecycleCon
     const dataSourceConfig: Record<string, DataSourceOptions> | undefined = (moduleConfigHolder.config as any).dataSource;
     if (!dataSourceConfig) return;
     await Promise.all(Object.entries(dataSourceConfig).map(async ([ name, config ]) => {
+      const dataSourceOptions = {
+        ...config,
+        name,
+      };
+      const forker = new DatabaseForker(this.env, dataSourceOptions);
+      if (forker.shouldFork()) {
+        await forker.forkDb(path.join(loadUnit.unitPath, 'dal'));
+      }
+
       try {
-        await MysqlDataSourceManager.instance.createDataSource(loadUnit.name, name, {
-          ...config,
-          name,
-        });
+        await MysqlDataSourceManager.instance.createDataSource(loadUnit.name, name, dataSourceOptions);
       } catch (e) {
         e.message = `create module ${loadUnit.name} datasource ${name} failed: ` + e.message;
         throw e;

plugin/dal/test/fixtures/apps/dal-app/config/config.default.js

@@ -6,7 +6,7 @@ module.exports = function() {
     security: {
       csrf: {
         ignoreJSON: false,
-      }
+      },
     },
   };
   return config;

plugin/dal/test/fixtures/apps/dal-app/modules/dal/module.yml

@@ -1,8 +1,9 @@
 dataSource:
   foo:
     connectionLimit: 100
-    database: 'test'
+    database: 'test_dal_plugin'
     host: '127.0.0.1'
     user: root
     port: 3306
     timezone: '+08:00'
+    forkDb: true

standalone/standalone/src/Runner.ts

@@ -154,7 +154,7 @@ export class Runner {
     this.loadUnitMultiInstanceProtoHook = new LoadUnitMultiInstanceProtoHook();
     LoadUnitLifecycleUtil.registerLifecycle(this.loadUnitMultiInstanceProtoHook);
 
-    this.dalModuleLoadUnitHook = new DalModuleLoadUnitHook(this.moduleConfigs);
+    this.dalModuleLoadUnitHook = new DalModuleLoadUnitHook(this.env ?? '', this.moduleConfigs);
     const loggerInnerObject = this.innerObjects.logger && this.innerObjects.logger[0];
     const logger = loggerInnerObject?.obj || console;
     this.dalTableEggPrototypeHook = new DalTableEggPrototypeHook(logger as Logger);

standalone/standalone/test/fixtures/dal-module/module.yml

@@ -1,8 +1,9 @@
 dataSource:
   foo:
     connectionLimit: 100
-    database: 'test'
+    database: 'test_dal_standalone'
     host: '127.0.0.1'
     user: root
     port: 3306
     timezone: '+08:00'
+    forkDb: true

standalone/standalone/test/index.test.ts

@@ -6,8 +6,6 @@ import { ModuleConfigs } from '@eggjs/tegg';
 import { ModuleConfig } from 'egg';
 import { crosscutAdviceParams, pointcutAdviceParams } from './fixtures/aop-module/Hello';
 import { Foo } from './fixtures/dal-module/Foo';
-import { MysqlDataSource, SqlGenerator } from '@eggjs/dal-runtime';
-import { TableModel } from '@eggjs/dal-decorator';
 
 describe('test/index.test.ts', () => {
   describe('simple runner', () => {
@@ -217,34 +215,10 @@ describe('test/index.test.ts', () => {
   });
 
   describe('dal runner', () => {
-    let mysqlDataSource: MysqlDataSource;
-
-    before(async () => {
-      mysqlDataSource = new MysqlDataSource({
-        name: 'foo',
-        host: '127.0.0.1',
-        user: 'root',
-        database: 'test',
-        timezone: '+08:00',
-        initSql: 'SET GLOBAL time_zone = \'+08:00\';',
-      });
-      await mysqlDataSource.ready();
-      await mysqlDataSource.query('DROP TABLE IF EXISTS egg_foo');
-
-      const tableModel = TableModel.build(Foo);
-
-      const sqlGenerator = new SqlGenerator();
-      const createTableSql = sqlGenerator.generate(tableModel);
-
-      await mysqlDataSource.query(createTableSql);
-    });
-
-    after(async () => {
-      await mysqlDataSource.query('DROP TABLE IF EXISTS egg_foo');
-    });
-
     it('should work', async () => {
-      const foo: Foo = await main(path.join(__dirname, './fixtures/dal-module'));
+      const foo: Foo = await main(path.join(__dirname, './fixtures/dal-module'), {
+        env: 'unittest',
+      });
       assert(foo);
       assert.equal(foo.col1, '2333');
     });