From a411f04e074425419b5b348a362f120bf8189541 Mon Sep 17 00:00:00 2001 From: killa Date: Tue, 2 Apr 2024 14:58:28 +0800 Subject: [PATCH] feat: impl dal forkDb (#202) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ##### Checklist - [ ] `npm test` passes - [ ] tests and/or benchmarks are included - [ ] documentation is changed or added - [ ] commit message follows commit guidelines ##### Affected core subsystem(s) ##### Description of change ## Summary by CodeRabbit - **New Features** - Introduced `DatabaseForker` for efficient database forking, particularly useful in unit testing environments. - Added configuration options for database forking in various modules and tests. - Enhanced `MysqlDataSource` with new properties for improved database connection handling. - **Documentation** - Updated README with instructions for unit test database configuration. - **Tests** - Adjusted database initialization and cleanup processes in unit tests to utilize `DatabaseForker`. - Modified database configurations in test fixtures to support new forking functionality. - **Refactor** - Refactored database connection and forking logic across multiple modules to integrate `DatabaseForker`. --- core/dal-runtime/index.ts | 1 + core/dal-runtime/src/DatabaseForker.ts | 68 +++++++++++++++++++ core/dal-runtime/src/MySqlDataSource.ts | 14 ++-- core/dal-runtime/test/DAO.test.ts | 25 ++++--- core/dal-runtime/test/DataSource.test.ts | 24 ++++--- plugin/dal/README.md | 12 ++++ plugin/dal/app.ts | 2 +- plugin/dal/lib/DalModuleLoadUnitHook.ts | 21 ++++-- .../apps/dal-app/config/config.default.js | 2 +- .../apps/dal-app/modules/dal/module.yml | 3 +- standalone/standalone/src/Runner.ts | 2 +- .../test/fixtures/dal-module/module.yml | 3 +- standalone/standalone/test/index.test.ts | 32 +-------- 13 files changed, 142 insertions(+), 67 deletions(-) create mode 100644 core/dal-runtime/src/DatabaseForker.ts diff --git a/core/dal-runtime/index.ts b/core/dal-runtime/index.ts index e7061137..975157be 100644 --- a/core/dal-runtime/index.ts +++ b/core/dal-runtime/index.ts @@ -6,3 +6,4 @@ export * from './src/SqlMapLoader'; export * from './src/DataSource'; export * from './src/MySqlDataSource'; export * from './src/TableModelInstanceBuilder'; +export * from './src/DatabaseForker'; diff --git a/core/dal-runtime/src/DatabaseForker.ts b/core/dal-runtime/src/DatabaseForker.ts new file mode 100644 index 00000000..ede373ce --- /dev/null +++ b/core/dal-runtime/src/DatabaseForker.ts @@ -0,0 +1,68 @@ +import { DataSourceOptions } from './MySqlDataSource'; +import { RDSClient } from '@eggjs/rds'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import assert from 'node:assert'; + +export class DatabaseForker { + private readonly env: string; + private readonly options: DataSourceOptions; + + constructor(env: string, options: DataSourceOptions) { + this.env = env; + this.options = options; + } + + shouldFork() { + return this.env === 'unittest' && this.options.forkDb; + } + + async forkDb(dalDir: string) { + assert(this.shouldFork(), 'fork db only run in unittest'); + // 尽早判断不应该 fork，避免对 rds pool 配置造成污染 + try { + await fs.access(dalDir); + } catch (_) { + return; + } + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { name, initSql, forkDb, database, ...mysqlOptions } = this.options; + const client = new RDSClient(Object.assign(mysqlOptions)); + const conn = await client.getConnection(); + await this.doCreateUtDb(conn); + await this.forkTables(conn, dalDir); + conn.release(); + await client.end(); + } + + private async forkTables(conn, dalDir: string) { + const sqlDir = path.join(dalDir, 'structure'); + const structureFiles = await fs.readdir(sqlDir); + const sqlFiles = structureFiles.filter(t => t.endsWith('.sql')); + for (const sqlFile of sqlFiles) { + await this.doForkTable(conn, path.join(sqlDir, sqlFile)); + } + } + + private async doForkTable(conn, sqlFileName: string) { + const sqlFile = await fs.readFile(sqlFileName, 'utf8'); + const sqls = sqlFile.split(';').filter(t => !!t.trim()); + for (const sql of sqls) { + await conn.query(sql); + } + } + + private async doCreateUtDb(conn) { + await conn.query(`CREATE DATABASE IF NOT EXISTS ${this.options.database};`); + await conn.query(`use ${this.options.database};`); + } + + async destroy() { + assert(this.shouldFork(), 'fork db only run in unittest'); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { name, initSql, forkDb, database, ...mysqlOptions } = this.options; + const client = new RDSClient(Object.assign(mysqlOptions)); + await client.query(`DROP DATABASE ${database}`); + await client.end(); + } +} diff --git a/core/dal-runtime/src/MySqlDataSource.ts b/core/dal-runtime/src/MySqlDataSource.ts index 25b1738e..e3e06d3b 100644 --- a/core/dal-runtime/src/MySqlDataSource.ts +++ b/core/dal-runtime/src/MySqlDataSource.ts @@ -1,12 +1,12 @@ import { RDSClient } from '@eggjs/rds'; -// TODO fix export -import type { RDSClientOptions } from '@eggjs/rds/lib/types'; +import type { RDSClientOptions } from '@eggjs/rds'; import Base from 'sdk-base'; export interface DataSourceOptions extends RDSClientOptions { name: string; // default is select 1 + 1; initSql?: string; + forkDb?: boolean; } const DEFAULT_OPTIONS: RDSClientOptions = { @@ -16,18 +16,22 @@ const DEFAULT_OPTIONS: RDSClientOptions = { }; export class MysqlDataSource extends Base { - private readonly client: RDSClient; + private client: RDSClient; private readonly initSql: string; readonly name: string; readonly timezone?: string; + readonly rdsOptions: RDSClientOptions; + readonly forkDb?: boolean; constructor(options: DataSourceOptions) { super({ initMethod: '_init' }); - const { name, initSql, ...mysqlOptions } = options; - this.client = new RDSClient(Object.assign({}, DEFAULT_OPTIONS, mysqlOptions)); + const { name, initSql, forkDb, ...mysqlOptions } = options; + this.forkDb = forkDb; this.initSql = initSql ?? 'SELECT 1 + 1'; this.name = name; this.timezone = options.timezone; + this.rdsOptions = Object.assign({}, DEFAULT_OPTIONS, mysqlOptions); + this.client = new RDSClient(this.rdsOptions); } protected async _init() { diff --git a/core/dal-runtime/test/DAO.test.ts b/core/dal-runtime/test/DAO.test.ts index 6b54a858..eff290ba 100644 --- a/core/dal-runtime/test/DAO.test.ts +++ b/core/dal-runtime/test/DAO.test.ts @@ -5,37 +5,40 @@ import { Foo } from './fixtures/modules/dal/Foo'; import { TableModel } from '@eggjs/dal-decorator'; import path from 'node:path'; import { DataSource } from '../src/DataSource'; -import { SqlGenerator } from '../src/SqlGenerator'; import FooDAO from './fixtures/modules/dal/dal/dao/FooDAO'; +import { DatabaseForker } from '../src/DatabaseForker'; describe('test/DAO.test.ts', () => { let dataSource: DataSource; let tableModel: TableModel; + let forker: DatabaseForker; before(async () => { - const mysql = new MysqlDataSource({ + const mysqlOptions = { name: 'foo', host: '127.0.0.1', user: 'root', - database: 'test', + database: 'test_runtime', timezone: '+08:00', initSql: 'SET GLOBAL time_zone = \'+08:00\';', - }); + forkDb: true, + }; + forker = new DatabaseForker('unittest', mysqlOptions); + await forker.forkDb(path.join(__dirname, './fixtures/modules/dal/dal')); + + const mysql = new MysqlDataSource(mysqlOptions); await mysql.ready(); - await mysql.query('DROP TABLE IF EXISTS egg_foo'); tableModel = TableModel.build(Foo); - - const sqlGenerator = new SqlGenerator(); - const createTableSql = sqlGenerator.generate(tableModel); - - await mysql.query(createTableSql); - const sqlMapLoader = new SqlMapLoader(tableModel, path.join(__dirname, './fixtures/modules/dal'), console as any); const sqlMap = sqlMapLoader.load(); dataSource = new DataSource(tableModel, mysql, sqlMap); }); + after(async () => { + await forker.destroy(); + }); + it('execute should work', async () => { const foo = new Foo(); foo.name = 'name'; diff --git a/core/dal-runtime/test/DataSource.test.ts b/core/dal-runtime/test/DataSource.test.ts index 2a6ab357..e4e2fda6 100644 --- a/core/dal-runtime/test/DataSource.test.ts +++ b/core/dal-runtime/test/DataSource.test.ts @@ -7,36 +7,38 @@ import path from 'node:path'; import { DataSource } from '../src/DataSource'; import { TableModelInstanceBuilder } from '../src/TableModelInstanceBuilder'; import { DeleteResult, InsertResult, UpdateResult } from '@eggjs/rds/lib/types'; -import { SqlGenerator } from '../src/SqlGenerator'; +import { DatabaseForker } from '../src/DatabaseForker'; describe('test/Datasource.test.ts', () => { let dataSource: DataSource; let tableModel: TableModel; + let forker: DatabaseForker; before(async () => { - const mysql = new MysqlDataSource({ + const mysqlOptions = { name: 'foo', host: '127.0.0.1', user: 'root', - database: 'test', + database: 'test_runtime', timezone: '+08:00', initSql: 'SET GLOBAL time_zone = \'+08:00\';', - }); + forkDb: true, + }; + forker = new DatabaseForker('unittest', mysqlOptions); + await forker.forkDb(path.join(__dirname, './fixtures/modules/dal/dal')); + const mysql = new MysqlDataSource(mysqlOptions); await mysql.ready(); - await mysql.query('DROP TABLE IF EXISTS egg_foo'); tableModel = TableModel.build(Foo); - - const sqlGenerator = new SqlGenerator(); - const createTableSql = sqlGenerator.generate(tableModel); - - await mysql.query(createTableSql); - const sqlMapLoader = new SqlMapLoader(tableModel, path.join(__dirname, './fixtures/modules/dal'), console as any); const sqlMap = sqlMapLoader.load(); dataSource = new DataSource(tableModel, mysql, sqlMap); }); + after(async () => { + await forker.destroy(); + }); + it('execute should work', async () => { const foo = new Foo(); foo.name = 'name'; diff --git a/plugin/dal/README.md b/plugin/dal/README.md index 05e33853..54089080 100644 --- a/plugin/dal/README.md +++ b/plugin/dal/README.md @@ -637,3 +637,15 @@ dataSource: ```sql SELECT @@GLOBAL.time_zone; ``` + +## Unittest +可以在 `module.yml` 中开启 forkDb 配置，即可实现 unittest 环境自动创建数据库 + +```yaml +# module.yml +dataSource: + foo: + # 开启 ci 环境自动创建数据库 + forkDb: true + +``` diff --git a/plugin/dal/app.ts b/plugin/dal/app.ts index 0f86c014..adb6b42b 100644 --- a/plugin/dal/app.ts +++ b/plugin/dal/app.ts @@ -17,7 +17,7 @@ export default class ControllerAppBootHook { } configWillLoad() { - this.dalModuleLoadUnitHook = new DalModuleLoadUnitHook(this.app.moduleConfigs); + this.dalModuleLoadUnitHook = new DalModuleLoadUnitHook(this.app.config.env, this.app.moduleConfigs); this.dalTableEggPrototypeHook = new DalTableEggPrototypeHook(this.app.logger); this.app.eggPrototypeLifecycleUtil.registerLifecycle(this.dalTableEggPrototypeHook); this.app.loadUnitLifecycleUtil.registerLifecycle(this.dalModuleLoadUnitHook); diff --git a/plugin/dal/lib/DalModuleLoadUnitHook.ts b/plugin/dal/lib/DalModuleLoadUnitHook.ts index f0ceec7e..b06b603b 100644 --- a/plugin/dal/lib/DalModuleLoadUnitHook.ts +++ b/plugin/dal/lib/DalModuleLoadUnitHook.ts @@ -1,13 +1,16 @@ import { MysqlDataSourceManager } from './MysqlDataSourceManager'; +import path from 'node:path'; import { LifecycleHook } from '@eggjs/tegg-lifecycle'; import { ModuleConfigHolder } from '@eggjs/tegg-common-util'; -import { DataSourceOptions } from '@eggjs/dal-runtime'; +import { DatabaseForker, DataSourceOptions } from '@eggjs/dal-runtime'; import { LoadUnit, LoadUnitLifecycleContext } from '@eggjs/tegg/helper'; export class DalModuleLoadUnitHook implements LifecycleHook { private readonly moduleConfigs: Record; + private readonly env: string; - constructor(moduleConfigs: Record) { + constructor(env: string, moduleConfigs: Record) { + this.env = env; this.moduleConfigs = moduleConfigs; } @@ -17,11 +20,17 @@ export class DalModuleLoadUnitHook implements LifecycleHook | undefined = (moduleConfigHolder.config as any).dataSource; if (!dataSourceConfig) return; await Promise.all(Object.entries(dataSourceConfig).map(async ([ name, config ]) => { + const dataSourceOptions = { + ...config, + name, + }; + const forker = new DatabaseForker(this.env, dataSourceOptions); + if (forker.shouldFork()) { + await forker.forkDb(path.join(loadUnit.unitPath, 'dal')); + } + try { - await MysqlDataSourceManager.instance.createDataSource(loadUnit.name, name, { - ...config, - name, - }); + await MysqlDataSourceManager.instance.createDataSource(loadUnit.name, name, dataSourceOptions); } catch (e) { e.message = `create module ${loadUnit.name} datasource ${name} failed: ` + e.message; throw e; diff --git a/plugin/dal/test/fixtures/apps/dal-app/config/config.default.js b/plugin/dal/test/fixtures/apps/dal-app/config/config.default.js index 586c9e0c..31ec8ab4 100644 --- a/plugin/dal/test/fixtures/apps/dal-app/config/config.default.js +++ b/plugin/dal/test/fixtures/apps/dal-app/config/config.default.js @@ -6,7 +6,7 @@ module.exports = function() { security: { csrf: { ignoreJSON: false, - } + }, }, }; return config; diff --git a/plugin/dal/test/fixtures/apps/dal-app/modules/dal/module.yml b/plugin/dal/test/fixtures/apps/dal-app/modules/dal/module.yml index 191f15f1..f47ca733 100644 --- a/plugin/dal/test/fixtures/apps/dal-app/modules/dal/module.yml +++ b/plugin/dal/test/fixtures/apps/dal-app/modules/dal/module.yml @@ -1,8 +1,9 @@ dataSource: foo: connectionLimit: 100 - database: 'test' + database: 'test_dal_plugin' host: '127.0.0.1' user: root port: 3306 timezone: '+08:00' + forkDb: true diff --git a/standalone/standalone/src/Runner.ts b/standalone/standalone/src/Runner.ts index 6ff7aeb9..bdc0cef5 100644 --- a/standalone/standalone/src/Runner.ts +++ b/standalone/standalone/src/Runner.ts @@ -154,7 +154,7 @@ export class Runner { this.loadUnitMultiInstanceProtoHook = new LoadUnitMultiInstanceProtoHook(); LoadUnitLifecycleUtil.registerLifecycle(this.loadUnitMultiInstanceProtoHook); - this.dalModuleLoadUnitHook = new DalModuleLoadUnitHook(this.moduleConfigs); + this.dalModuleLoadUnitHook = new DalModuleLoadUnitHook(this.env ?? '', this.moduleConfigs); const loggerInnerObject = this.innerObjects.logger && this.innerObjects.logger[0]; const logger = loggerInnerObject?.obj || console; this.dalTableEggPrototypeHook = new DalTableEggPrototypeHook(logger as Logger); diff --git a/standalone/standalone/test/fixtures/dal-module/module.yml b/standalone/standalone/test/fixtures/dal-module/module.yml index 191f15f1..9d504b8c 100644 --- a/standalone/standalone/test/fixtures/dal-module/module.yml +++ b/standalone/standalone/test/fixtures/dal-module/module.yml @@ -1,8 +1,9 @@ dataSource: foo: connectionLimit: 100 - database: 'test' + database: 'test_dal_standalone' host: '127.0.0.1' user: root port: 3306 timezone: '+08:00' + forkDb: true diff --git a/standalone/standalone/test/index.test.ts b/standalone/standalone/test/index.test.ts index 8b3a3302..e1b21db9 100644 --- a/standalone/standalone/test/index.test.ts +++ b/standalone/standalone/test/index.test.ts @@ -6,8 +6,6 @@ import { ModuleConfigs } from '@eggjs/tegg'; import { ModuleConfig } from 'egg'; import { crosscutAdviceParams, pointcutAdviceParams } from './fixtures/aop-module/Hello'; import { Foo } from './fixtures/dal-module/Foo'; -import { MysqlDataSource, SqlGenerator } from '@eggjs/dal-runtime'; -import { TableModel } from '@eggjs/dal-decorator'; describe('test/index.test.ts', () => { describe('simple runner', () => { @@ -217,34 +215,10 @@ describe('test/index.test.ts', () => { }); describe('dal runner', () => { - let mysqlDataSource: MysqlDataSource; - - before(async () => { - mysqlDataSource = new MysqlDataSource({ - name: 'foo', - host: '127.0.0.1', - user: 'root', - database: 'test', - timezone: '+08:00', - initSql: 'SET GLOBAL time_zone = \'+08:00\';', - }); - await mysqlDataSource.ready(); - await mysqlDataSource.query('DROP TABLE IF EXISTS egg_foo'); - - const tableModel = TableModel.build(Foo); - - const sqlGenerator = new SqlGenerator(); - const createTableSql = sqlGenerator.generate(tableModel); - - await mysqlDataSource.query(createTableSql); - }); - - after(async () => { - await mysqlDataSource.query('DROP TABLE IF EXISTS egg_foo'); - }); - it('should work', async () => { - const foo: Foo = await main(path.join(__dirname, './fixtures/dal-module')); + const foo: Foo = await main(path.join(__dirname, './fixtures/dal-module'), { + env: 'unittest', + }); assert(foo); assert.equal(foo.col1, '2333'); });