From aa2953172b6099c1059597c4737f8b3b3466e673 Mon Sep 17 00:00:00 2001 From: ouyangxq <775153912@qq.com> Date: Sat, 19 Sep 2026 17:30:20 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deploy/sql/README.md | 24 ++ deploy/sql/migration-scopes.json | 44 +++ .../sql/migrations/cn/baiyue_audit/.gitkeep | 1 + deploy/sql/migrations/cn/baiyue_auth/.gitkeep | 1 + .../cn/baiyue_event_bridge/.gitkeep | 1 + deploy/sql/migrations/cn/baiyue_file/.gitkeep | 1 + .../sql/migrations/cn/baiyue_finance/.gitkeep | 1 + .../migrations/cn/baiyue_firstleg/.gitkeep | 1 + deploy/sql/migrations/cn/baiyue_iam/.gitkeep | 1 + .../sql/migrations/cn/baiyue_order/.gitkeep | 1 + .../migrations/cn/baiyue_warehouse/.gitkeep | 1 + .../baiyue_event_bridge_overseas/.gitkeep | 1 + .../baiyue_lastlag_warehouse/.gitkeep | 1 + .../overseas/baiyue_lastleg/.gitkeep | 1 + deploy/sql/scripts/db-migrations.mjs | 322 ++++++++++++++++++ deploy/sql/scripts/db-migrations.test.mjs | 77 +++++ deploy/sql/数据库迁移操作手册.md | 141 ++++++++ .../FinanceSqlCommentContractTest.java | 2 +- scripts/README.md | 2 + .../script-documentation-contract.test.mjs | 2 + 20 files changed, 625 insertions(+), 1 deletion(-) create mode 100644 deploy/sql/migration-scopes.json create mode 100644 deploy/sql/migrations/cn/baiyue_audit/.gitkeep create mode 100644 deploy/sql/migrations/cn/baiyue_auth/.gitkeep create mode 100644 deploy/sql/migrations/cn/baiyue_event_bridge/.gitkeep create mode 100644 deploy/sql/migrations/cn/baiyue_file/.gitkeep create mode 100644 deploy/sql/migrations/cn/baiyue_finance/.gitkeep create mode 100644 deploy/sql/migrations/cn/baiyue_firstleg/.gitkeep create mode 100644 deploy/sql/migrations/cn/baiyue_iam/.gitkeep create mode 100644 deploy/sql/migrations/cn/baiyue_order/.gitkeep create mode 100644 deploy/sql/migrations/cn/baiyue_warehouse/.gitkeep create mode 100644 deploy/sql/migrations/overseas/baiyue_event_bridge_overseas/.gitkeep create mode 100644 deploy/sql/migrations/overseas/baiyue_lastlag_warehouse/.gitkeep create mode 100644 deploy/sql/migrations/overseas/baiyue_lastleg/.gitkeep create mode 100644 deploy/sql/scripts/db-migrations.mjs create mode 100644 deploy/sql/scripts/db-migrations.test.mjs create mode 100644 deploy/sql/数据库迁移操作手册.md diff --git a/deploy/sql/README.md b/deploy/sql/README.md index 087e52ea9..eb6d0c0e5 100644 --- a/deploy/sql/README.md +++ b/deploy/sql/README.md @@ -60,6 +60,30 @@ - 模块 `src/main/resources/db/` 用作结构来源和开发追溯，不是区域部署前置；不再向历史迁移文件追加发布步骤。修改当前字段时同步模块有效建表定义及其合同。 - 不在 SQL 中写入真实客户数据、环境地址、固定密码或 Token。 +## 后续迁移工具（已开始实施） + +机制启用前，以上两份统一 SQL 仍是空库初始化的唯一入口；本节工具不会修改已有数据库，也不会把当前可丢弃的开发/测试存量转换成迁移链。 + +迁移范围映射位于 `migration-scopes.json`，按区域、schema、所属服务和顺序登记。迁移文件位于 `migrations/{cn|overseas}//`，一个文件只允许修改一个 schema。目录中的 `.gitkeep` 仅用于保留空目录，不是迁移文件。 + +完整操作请先读 [数据库迁移操作手册](数据库迁移操作手册.md)。离线入口为 `scripts/db-migrations.mjs`： + +```powershell +# 自动取得该 schema 的下一个整数序号并创建草稿；不连接数据库 +node deploy/sql/scripts/db-migrations.mjs new --region CN --schema baiyue_auth --description add_login_audit + +# 校验范围、文件名、跨 schema 引用及受保护文件；不连接数据库 +node deploy/sql/scripts/db-migrations.mjs check + +# 由选中服务自动生成逐库目标和 SQL 哈希；不手填发布 JSON +node deploy/sql/scripts/db-migrations.mjs plan --services auth,iam --regions CN --out release-manifest.json + +# 当前只提供安全的只读计划；实际 Flyway 执行需后续阶段显式配置并授权 +node deploy/sql/scripts/db-migrations.mjs run --manifest release-manifest.json --dry-run +``` + +`new` 使用独占创建避免并发覆盖；成功历史或已冻结制品不能由普通命令改写。`plan` 只打包选中服务及评审确认的 schema，不会因为同一提交存在其他区域或服务 SQL 就自动扩大范围。`run` 的真实执行还未接入测试数据库和 Jenkins，当前未授权任何部署或 SQL 操作。 + ## 本地验证 离线合同不连接数据库： diff --git a/deploy/sql/migration-scopes.json b/deploy/sql/migration-scopes.json new file mode 100644 index 000000000..ebe28e67e --- /dev/null +++ b/deploy/sql/migration-scopes.json @@ -0,0 +1,44 @@ +{ + "schemaVersion": 1, + "services": [ + "gateway", + "gateway-overseas", + "auth", + "iam", + "order", + "warehouse", + "firstleg", + "file", + "audit-service", + "event-bridge-service", + "finance", + "integration-service", + "print", + "lastleg-service", + "warehouse-overseas-service" + ], + "regions": { + "CN": { + "baseline": "baseline/cn/cn-baseline.sql", + "schemas": [ + { "name": "baiyue_auth", "services": ["auth"], "order": 10 }, + { "name": "baiyue_iam", "services": ["iam"], "order": 20 }, + { "name": "baiyue_order", "services": ["order"], "order": 30 }, + { "name": "baiyue_warehouse", "services": ["warehouse"], "order": 40 }, + { "name": "baiyue_firstleg", "services": ["firstleg"], "order": 50 }, + { "name": "baiyue_file", "services": ["file"], "order": 60 }, + { "name": "baiyue_audit", "services": ["audit-service"], "order": 70 }, + { "name": "baiyue_event_bridge", "services": ["event-bridge-service"], "order": 80 }, + { "name": "baiyue_finance", "services": ["finance"], "order": 90 } + ] + }, + "OVERSEAS": { + "baseline": "baseline/overseas/overseas-baseline.sql", + "schemas": [ + { "name": "baiyue_lastleg", "services": ["lastleg-service"], "order": 10 }, + { "name": "baiyue_lastlag_warehouse", "services": ["warehouse-overseas-service"], "order": 20 }, + { "name": "baiyue_event_bridge_overseas", "services": ["event-bridge-service"], "order": 30 } + ] + } + } +} diff --git a/deploy/sql/migrations/cn/baiyue_audit/.gitkeep b/deploy/sql/migrations/cn/baiyue_audit/.gitkeep new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/deploy/sql/migrations/cn/baiyue_audit/.gitkeep @@ -0,0 +1 @@ + diff --git a/deploy/sql/migrations/cn/baiyue_auth/.gitkeep b/deploy/sql/migrations/cn/baiyue_auth/.gitkeep new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/deploy/sql/migrations/cn/baiyue_auth/.gitkeep @@ -0,0 +1 @@ + diff --git a/deploy/sql/migrations/cn/baiyue_event_bridge/.gitkeep b/deploy/sql/migrations/cn/baiyue_event_bridge/.gitkeep new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/deploy/sql/migrations/cn/baiyue_event_bridge/.gitkeep @@ -0,0 +1 @@ + diff --git a/deploy/sql/migrations/cn/baiyue_file/.gitkeep b/deploy/sql/migrations/cn/baiyue_file/.gitkeep new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/deploy/sql/migrations/cn/baiyue_file/.gitkeep @@ -0,0 +1 @@ + diff --git a/deploy/sql/migrations/cn/baiyue_finance/.gitkeep b/deploy/sql/migrations/cn/baiyue_finance/.gitkeep new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/deploy/sql/migrations/cn/baiyue_finance/.gitkeep @@ -0,0 +1 @@ + diff --git a/deploy/sql/migrations/cn/baiyue_firstleg/.gitkeep b/deploy/sql/migrations/cn/baiyue_firstleg/.gitkeep new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/deploy/sql/migrations/cn/baiyue_firstleg/.gitkeep @@ -0,0 +1 @@ + diff --git a/deploy/sql/migrations/cn/baiyue_iam/.gitkeep b/deploy/sql/migrations/cn/baiyue_iam/.gitkeep new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/deploy/sql/migrations/cn/baiyue_iam/.gitkeep @@ -0,0 +1 @@ + diff --git a/deploy/sql/migrations/cn/baiyue_order/.gitkeep b/deploy/sql/migrations/cn/baiyue_order/.gitkeep new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/deploy/sql/migrations/cn/baiyue_order/.gitkeep @@ -0,0 +1 @@ + diff --git a/deploy/sql/migrations/cn/baiyue_warehouse/.gitkeep b/deploy/sql/migrations/cn/baiyue_warehouse/.gitkeep new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/deploy/sql/migrations/cn/baiyue_warehouse/.gitkeep @@ -0,0 +1 @@ + diff --git a/deploy/sql/migrations/overseas/baiyue_event_bridge_overseas/.gitkeep b/deploy/sql/migrations/overseas/baiyue_event_bridge_overseas/.gitkeep new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/deploy/sql/migrations/overseas/baiyue_event_bridge_overseas/.gitkeep @@ -0,0 +1 @@ + diff --git a/deploy/sql/migrations/overseas/baiyue_lastlag_warehouse/.gitkeep b/deploy/sql/migrations/overseas/baiyue_lastlag_warehouse/.gitkeep new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/deploy/sql/migrations/overseas/baiyue_lastlag_warehouse/.gitkeep @@ -0,0 +1 @@ + diff --git a/deploy/sql/migrations/overseas/baiyue_lastleg/.gitkeep b/deploy/sql/migrations/overseas/baiyue_lastleg/.gitkeep new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/deploy/sql/migrations/overseas/baiyue_lastleg/.gitkeep @@ -0,0 +1 @@ + diff --git a/deploy/sql/scripts/db-migrations.mjs b/deploy/sql/scripts/db-migrations.mjs new file mode 100644 index 000000000..ec533761b --- /dev/null +++ b/deploy/sql/scripts/db-migrations.mjs @@ -0,0 +1,322 @@ +#!/usr/bin/env node + +import { createHash } from 'node:crypto' +import { mkdir, open, readFile, readdir, stat, writeFile } from 'node:fs/promises' +import { spawnSync } from 'node:child_process' +import path from 'node:path' +import process from 'node:process' +import { fileURLToPath } from 'node:url' + +const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url)) +const SQL_ROOT = path.resolve(SCRIPT_DIR, '..') +const SCOPES_PATH = path.join(SQL_ROOT, 'migration-scopes.json') +const MIGRATION_NAME = /^V(\d+)__([a-z0-9][a-z0-9_-]*)\.sql$/ + +function fail(message) { + throw new Error(message) +} + +function parseArgs(argv) { + const [command = 'help', ...rest] = argv + const args = { command, _: [] } + for (let index = 0; index < rest.length; index += 1) { + const item = rest[index] + if (!item.startsWith('--')) { + args._.push(item) + continue + } + const equals = item.indexOf('=') + if (equals >= 0) { + args[item.slice(2, equals)] = item.slice(equals + 1) + continue + } + const key = item.slice(2) + const next = rest[index + 1] + if (next && !next.startsWith('--')) { + args[key] = next + index += 1 + } else { + args[key] = true + } + } + return args +} + +function absoluteFrom(root, value) { + return path.isAbsolute(value) ? value : path.resolve(root, value) +} + +async function loadScopes(scopesPath = SCOPES_PATH) { + const scopes = JSON.parse(await readFile(scopesPath, 'utf8')) + if (scopes.schemaVersion !== 1 || !scopes.regions || typeof scopes.regions !== 'object') { + fail('migration-scopes.json 必须使用 schemaVersion=1 并提供 regions') + } + const seenSchemas = new Set() + for (const [region, config] of Object.entries(scopes.regions)) { + if (!/^[A-Z][A-Z0-9_]*$/.test(region)) fail(`区域名非法：${region}`) + if (!Array.isArray(config.schemas) || config.schemas.length === 0) fail(`${region} 没有 schema 清单`) + const orders = new Set() + for (const schema of config.schemas) { + if (!/^[a-z][a-z0-9_]*$/.test(schema.name)) fail(`schema 名称非法：${schema.name}`) + if (seenSchemas.has(schema.name)) fail(`schema 重复登记：${schema.name}`) + if (!Number.isInteger(schema.order) || schema.order < 1 || orders.has(schema.order)) fail(`${region} 的 schema order 非法或重复：${schema.name}`) + if (!Array.isArray(schema.services) || schema.services.length === 0) fail(`${schema.name} 没有所属服务`) + seenSchemas.add(schema.name) + orders.add(schema.order) + } + } + return scopes +} + +function regionConfig(scopes, region) { + const value = scopes.regions[region] + if (!value) fail(`未知区域：${region}；允许值：${Object.keys(scopes.regions).join(', ')}`) + return value +} + +function schemaConfig(scopes, region, schemaName) { + const config = regionConfig(scopes, region) + const schema = config.schemas.find((item) => item.name === schemaName) + if (!schema) fail(`${region} 未登记 schema：${schemaName}`) + return schema +} + +async function listMigrations(sqlRoot, region, schemaName) { + const directory = path.join(sqlRoot, 'migrations', region.toLowerCase(), schemaName) + let entries = [] + try { + entries = await readdir(directory, { withFileTypes: true }) + } catch (error) { + if (error.code === 'ENOENT') return [] + throw error + } + const migrations = [] + for (const entry of entries) { + if (!entry.isFile() || entry.name.startsWith('.')) continue + const match = entry.name.match(MIGRATION_NAME) + if (!match) fail(`${path.relative(sqlRoot, path.join(directory, entry.name))} 文件名不符合 V__description.sql`) + const version = Number(match[1]) + if (!Number.isSafeInteger(version) || version < 1) fail(`迁移版本非法：${entry.name}`) + migrations.push({ + version, + description: match[2], + fileName: entry.name, + relativePath: path.relative(sqlRoot, path.join(directory, entry.name)).replaceAll(path.sep, '/'), + absolutePath: path.join(directory, entry.name), + }) + } + migrations.sort((left, right) => left.version - right.version || left.fileName.localeCompare(right.fileName)) + for (let index = 1; index < migrations.length; index += 1) { + if (migrations[index - 1].version === migrations[index].version) fail(`${region}/${schemaName} 版本重复：${migrations[index].version}`) + } + return migrations +} + +async function sha256(filePath) { + return createHash('sha256').update(await readFile(filePath)).digest('hex') +} + +function selectedServices(value) { + if (!value || value === 'ALL') return null + const services = value.split(',').map((item) => item.trim()).filter(Boolean) + if (services.length === 0) fail('--services 不能为空') + return [...new Set(services)] +} + +function selectSchemas(scopes, region, services) { + const config = regionConfig(scopes, region) + const selected = services === null + ? config.schemas + : config.schemas.filter((schema) => schema.services.some((service) => services.includes(service))) + return [...selected].sort((left, right) => left.order - right.order) +} + +async function buildPlan({ sqlRoot = SQL_ROOT, scopes, regions, services, commit = process.env.GIT_COMMIT ?? null }) { + const knownServices = new Set(scopes.services ?? Object.values(scopes.regions).flatMap((config) => config.schemas.flatMap((schema) => schema.services))) + const unknownServices = (services ?? []).filter((service) => !knownServices.has(service)) + if (unknownServices.length) fail(`范围映射中未登记服务：${unknownServices.join(', ')}`) + const plan = { + schemaVersion: 1, + sourceCommit: commit, + services: services ?? ['ALL'], + regions: [], + } + for (const region of regions) { + const config = regionConfig(scopes, region) + const schemas = [] + for (const schema of selectSchemas(scopes, region, services)) { + const migrations = await listMigrations(sqlRoot, region, schema.name) + schemas.push({ + schema: schema.name, + services: schema.services, + order: schema.order, + migrationDirectory: `migrations/${region.toLowerCase()}/${schema.name}`, + baseline: config.baseline, + target: migrations.length ? String(migrations.at(-1).version) : '0.0.0', + migrations: await Promise.all(migrations.map(async (migration) => ({ + version: String(migration.version), + description: migration.description, + path: migration.relativePath, + sha256: await sha256(migration.absolutePath), + }))), + }) + } + plan.regions.push({ region, schemas }) + } + return plan +} + +async function createMigration({ sqlRoot = SQL_ROOT, scopes, region, schemaName, description }) { + const schema = schemaConfig(scopes, region, schemaName) + const directory = path.join(sqlRoot, 'migrations', region.toLowerCase(), schema.name) + await mkdir(directory, { recursive: true }) + const normalizedDescription = description.trim() + if (!normalizedDescription) fail('必须填写变更说明') + const safeDescription = normalizedDescription.toLowerCase().replaceAll(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, '') || 'change' + for (let attempt = 0; attempt < 20; attempt += 1) { + const migrations = await listMigrations(sqlRoot, region, schema.name) + const version = (migrations.at(-1)?.version ?? 0) + 1 + const fileName = `V${version}__${safeDescription}.sql` + const filePath = path.join(directory, fileName) + try { + const handle = await open(filePath, 'wx') + await handle.writeFile([ + `-- Region: ${region}`, + `-- Schema: ${schema.name}`, + `-- Purpose: ${description.trim()}`, + '-- Preconditions:', + '-- Postconditions:', + '', + ].join('\n'), 'utf8') + await handle.close() + return { region, schema: schema.name, version, fileName, path: path.relative(sqlRoot, filePath).replaceAll(path.sep, '/') } + } catch (error) { + if (error.code !== 'EEXIST') throw error + } + } + fail(`${region}/${schema.name} 并发生成迁移失败：重试次数已耗尽`) +} + +async function checkProtected(sqlRoot, protectedPath) { + if (!protectedPath) return [] + const document = JSON.parse(await readFile(protectedPath, 'utf8')) + const entries = document.files ?? document.migrations ?? [] + if (!Array.isArray(entries)) fail('保护清单的 files/migrations 必须是数组') + const errors = [] + for (const entry of entries) { + if (!['success', 'frozen'].includes(entry.status)) continue + if (!entry.path || !/^[a-f0-9]{64}$/.test(entry.sha256 ?? '')) fail('保护清单条目必须包含 path 和 64 位 sha256') + const current = await sha256(absoluteFrom(sqlRoot, entry.path)) + if (current !== entry.sha256) errors.push(`受保护迁移已修改：${entry.path}`) + } + return errors +} + +async function checkSqlFiles(sqlRoot, scopes, regions) { + const errors = [] + const schemasByName = new Map() + for (const region of regions) { + for (const schema of regionConfig(scopes, region).schemas) schemasByName.set(schema.name, { region, schema }) + } + for (const [schemaName, { region }] of schemasByName) { + const migrations = await listMigrations(sqlRoot, region, schemaName) + for (const migration of migrations) { + const sql = await readFile(migration.absolutePath, 'utf8') + for (const candidate of schemasByName.keys()) { + if (candidate !== schemaName && new RegExp(`(?:\\b|\`)${candidate}(?:\\b|\`)`, 'i').test(sql)) { + errors.push(`${migration.relativePath} 引用了其他 schema：${candidate}`) + } + } + if (/\b(?:DROP\s+DATABASE|CREATE\s+DATABASE|USE\s+)/i.test(sql)) errors.push(`${migration.relativePath} 不得包含跨库控制语句`) + } + } + return errors +} + +async function check({ sqlRoot = SQL_ROOT, scopes, regions = Object.keys(scopes.regions), protectedPath }) { + const errors = await checkSqlFiles(sqlRoot, scopes, regions) + errors.push(...await checkProtected(sqlRoot, protectedPath)) + if (errors.length) fail(errors.join('\n')) + return { ok: true, regions, checkedAt: new Date().toISOString() } +} + +async function runManifest({ sqlRoot = SQL_ROOT, manifestPath, dryRun = false }) { + const manifest = JSON.parse(await readFile(manifestPath, 'utf8')) + if (manifest.schemaVersion !== 1 || !Array.isArray(manifest.regions)) fail('发布清单格式不受支持') + const commands = [] + for (const region of manifest.regions) { + for (const schema of region.schemas ?? []) { + if (schema.target === '0.0.0' && (schema.migrations ?? []).length === 0) continue + const location = path.join(sqlRoot, schema.migrationDirectory) + await stat(location).catch(() => fail(`迁移目录不存在：${schema.migrationDirectory}`)) + commands.push({ region: region.region, schema: schema.schema, target: schema.target, location }) + } + } + if (dryRun) return { mode: 'dry-run', commands } + if (process.env.DB_MIGRATION_ALLOW !== '1') fail('实际数据库执行被保护：先设置 DB_MIGRATION_ALLOW=1，并显式提供测试环境凭据') + if (!process.env.FLYWAY_BIN) fail('实际数据库执行需要 FLYWAY_BIN，阶段 B 只支持 --dry-run') + const results = [] + for (const command of commands) { + const result = spawnSync(process.env.FLYWAY_BIN, [ + '-locations=filesystem:' + command.location, + '-schemas=' + command.schema, + '-defaultSchema=' + command.schema, + '-table=flyway_schema_history', + '-baselineOnMigrate=false', + '-cleanDisabled=true', + '-outOfOrder=false', + '-validateOnMigrate=true', + '-target=' + command.target, + 'migrate', + ], { stdio: 'inherit', env: process.env }) + if (result.status !== 0) fail(`${command.region}/${command.schema} Flyway 执行失败，退出码 ${result.status}`) + results.push(command) + } + return { mode: 'execute', commands: results } +} + +function usage() { + console.log(`用法：\n node deploy/sql/scripts/db-migrations.mjs new --region CN --schema baiyue_auth --description add_login_audit\n node deploy/sql/scripts/db-migrations.mjs check [--protected ]\n node deploy/sql/scripts/db-migrations.mjs plan --services auth,iam --regions CN,OVERSEAS --out \n node deploy/sql/scripts/db-migrations.mjs run --manifest --dry-run`) +} + +async function main(argv = process.argv.slice(2)) { + const args = parseArgs(argv) + const scopesPath = args.scopes ? absoluteFrom(process.cwd(), args.scopes) : SCOPES_PATH + const scopes = await loadScopes(scopesPath) + const sqlRoot = path.dirname(scopesPath) + if (args.command === 'help') return usage() + if (args.command === 'new') { + const result = await createMigration({ sqlRoot, scopes, region: String(args.region ?? ''), schemaName: String(args.schema ?? ''), description: String(args.description ?? '') }) + console.log(JSON.stringify(result, null, 2)) + return + } + if (args.command === 'check') { + const result = await check({ sqlRoot, scopes, protectedPath: args.protected ? absoluteFrom(process.cwd(), args.protected) : undefined }) + console.log(JSON.stringify(result, null, 2)) + return + } + if (args.command === 'plan') { + const regions = String(args.regions ?? Object.keys(scopes.regions).join(',')).split(',').map((item) => item.trim()).filter(Boolean) + const plan = await buildPlan({ sqlRoot, scopes, regions, services: selectedServices(args.services) }) + const output = JSON.stringify(plan, null, 2) + '\n' + if (args.out) await writeFile(absoluteFrom(process.cwd(), args.out), output, 'utf8') + else process.stdout.write(output) + return + } + if (args.command === 'run') { + const result = await runManifest({ sqlRoot, manifestPath: absoluteFrom(process.cwd(), String(args.manifest ?? '')), dryRun: Boolean(args['dry-run']) }) + console.log(JSON.stringify(result, null, 2)) + return + } + usage() + fail(`未知命令：${args.command}`) +} + +export { buildPlan, check, createMigration, listMigrations as discoverMigrations, loadScopes, parseArgs, runManifest } + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch((error) => { + console.error(`ERROR: ${error.message}`) + process.exitCode = 1 + }) +} diff --git a/deploy/sql/scripts/db-migrations.test.mjs b/deploy/sql/scripts/db-migrations.test.mjs new file mode 100644 index 000000000..0a07ce4c5 --- /dev/null +++ b/deploy/sql/scripts/db-migrations.test.mjs @@ -0,0 +1,77 @@ +import assert from 'node:assert/strict' +import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' +import test from 'node:test' +import { + buildPlan, + check, + createMigration, + discoverMigrations, + loadScopes, + runManifest, +} from './db-migrations.mjs' + +async function fixture() { + const root = await mkdtemp(path.join(os.tmpdir(), 'db-migrations-')) + const sqlRoot = path.join(root, 'sql') + await mkdir(path.join(sqlRoot, 'migrations', 'cn', 'baiyue_auth'), { recursive: true }) + await writeFile(path.join(sqlRoot, 'scopes.json'), JSON.stringify({ + schemaVersion: 1, + regions: { + CN: { + baseline: 'baseline/cn.sql', + schemas: [ + { name: 'baiyue_auth', services: ['auth'], order: 10 }, + { name: 'baiyue_order', services: ['order'], order: 20 }, + ], + }, + }, + }), 'utf8') + return { root, sqlRoot, scopesPath: path.join(sqlRoot, 'scopes.json') } +} + +test('new 自动分配 schema 内下一个版本，并且可重复执行检查', async (t) => { + const context = await fixture() + t.after(() => rm(context.root, { recursive: true, force: true })) + const scopes = await loadScopes(context.scopesPath) + const first = await createMigration({ sqlRoot: context.sqlRoot, scopes, region: 'CN', schemaName: 'baiyue_auth', description: 'Add login audit' }) + assert.equal(first.fileName, 'V1__add_login_audit.sql') + const second = await createMigration({ sqlRoot: context.sqlRoot, scopes, region: 'CN', schemaName: 'baiyue_auth', description: 'add_session_index' }) + assert.equal(second.fileName, 'V2__add_session_index.sql') + assert.match(await readFile(path.join(context.sqlRoot, first.path), 'utf8'), /Schema: baiyue_auth/) + assert.equal((await discoverMigrations(context.sqlRoot, 'CN', 'baiyue_auth')).length, 2) +}) + +test('plan 只包含选中服务，不把同提交其他 schema 的 SQL 带入制品', async (t) => { + const context = await fixture() + t.after(() => rm(context.root, { recursive: true, force: true })) + const scopes = await loadScopes(context.scopesPath) + const created = await createMigration({ sqlRoot: context.sqlRoot, scopes, region: 'CN', schemaName: 'baiyue_auth', description: 'add_login_audit' }) + const plan = await buildPlan({ sqlRoot: context.sqlRoot, scopes, regions: ['CN'], services: ['auth'], commit: 'abc1234' }) + assert.equal(plan.sourceCommit, 'abc1234') + assert.deepEqual(plan.regions[0].schemas.map((schema) => schema.schema), ['baiyue_auth']) + assert.equal(plan.regions[0].schemas[0].target, '1') + assert.equal(plan.regions[0].schemas[0].migrations[0].path, created.path) +}) + +test('check 拒绝跨 schema 引用和跨库控制语句', async (t) => { + const context = await fixture() + t.after(() => rm(context.root, { recursive: true, force: true })) + const scopes = await loadScopes(context.scopesPath) + await writeFile(path.join(context.sqlRoot, 'migrations', 'cn', 'baiyue_auth', 'V1__bad.sql'), 'ALTER TABLE baiyue_order.t ADD c INT;\nUSE baiyue_order;\n', 'utf8') + await assert.rejects(() => check({ sqlRoot: context.sqlRoot, scopes, regions: ['CN'] }), /其他 schema|跨库控制语句/) +}) + +test('run 默认只能 dry-run，不能无意连接数据库', async (t) => { + const context = await fixture() + t.after(() => rm(context.root, { recursive: true, force: true })) + const scopes = await loadScopes(context.scopesPath) + await createMigration({ sqlRoot: context.sqlRoot, scopes, region: 'CN', schemaName: 'baiyue_auth', description: 'add_login_audit' }) + const plan = await buildPlan({ sqlRoot: context.sqlRoot, scopes, regions: ['CN'], services: ['auth'] }) + const manifestPath = path.join(context.root, 'release-manifest.json') + await writeFile(manifestPath, JSON.stringify(plan), 'utf8') + const dryRun = await runManifest({ sqlRoot: context.sqlRoot, manifestPath, dryRun: true }) + assert.equal(dryRun.mode, 'dry-run') + await assert.rejects(() => runManifest({ sqlRoot: context.sqlRoot, manifestPath }), /DB_MIGRATION_ALLOW/) +}) diff --git a/deploy/sql/数据库迁移操作手册.md b/deploy/sql/数据库迁移操作手册.md new file mode 100644 index 000000000..99b306ee9 --- /dev/null +++ b/deploy/sql/数据库迁移操作手册.md @@ -0,0 +1,141 @@ +# 数据库迁移操作手册 + +本文是开发、测试和发布人员的日常操作入口。设计原则和生产边界见仓库文档 [数据库版本迁移与 Jenkins 自动发布方案](../../../docs/数据库版本迁移与Jenkins自动发布方案.md)；当前阶段的初始化事实见本目录 [README.md](README.md)。 + +## 先分清两类 SQL + +| 类型 | 位置 | 用途 | 能否在已有业务库重复执行 | +|---|---|---|---| +| 空库初始化 | `deploy/sql/头程数据库统一增量-MySQL8.sql`、`海外区域统一增量-MySQL8.sql` | 初始化一个不存在或获授权清空的区域数据库 | 不能 | +| 版本迁移 | `deploy/sql/migrations/{cn|overseas}//V<数字>__<说明>.sql` | 机制启用后对已有数据库逐步升级 | 由 Flyway 按历史自动跳过已成功版本 | + +当前测试分支只完成了版本迁移的离线工具和目录骨架。没有冻结基线、没有具体业务变更时，不要复制初始化 SQL 生成 `V1`，也不要连接测试或生产数据库试跑。 + +## 日常开发：新建和修改迁移 + +以下命令均从 `baiyue-firstleg` 根目录执行，PowerShell 和 Bash 都可以使用 Node 命令。 + +### 1. 确认所属区域和 schema + +打开 [migration-scopes.json](migration-scopes.json)，只选择已登记的区域和 schema。服务没有数据库时不创建空迁移；例如 `gateway` 不对应 schema。 + +```powershell +node deploy/sql/scripts/db-migrations.mjs check +``` + +### 2. 自动创建迁移文件 + +```powershell +node deploy/sql/scripts/db-migrations.mjs new ` + --region CN ` + --schema baiyue_auth ` + --description add_login_audit +``` + +工具会读取该 schema 当前最大编号并创建下一个文件，例如： + +```text +deploy/sql/migrations/cn/baiyue_auth/V1__add_login_audit.sql +``` + +开发者只输入区域、schema 和说明，不输入版本号。说明中的英文字母、数字和下划线会用于文件名；只有中文说明时文件名使用 `change`，完整中文说明仍保留在 SQL 头部。并发创建使用独占文件创建，不覆盖已有文件。 + +### 3. 填写 SQL 和断言 + +在生成的文件中完成实际 SQL，并补充文件头的前置、后置检查。一个文件只修改一个 schema，不写 `USE`、`CREATE DATABASE`、跨 schema 限定名或真实业务数据。 + +同一文件在开发阶段可以反复编辑，但一旦在保留历史的环境成功执行或进入冻结候选，就不能再改。没有成功记录但测试候选已失败时，只有方案第 13.2.1 节的受控修订流程可以保留原编号修正。 + +### 4. 提交前离线检查 + +```powershell +node deploy/sql/scripts/db-migrations.mjs check +node --test deploy/sql/scripts/db-migrations.test.mjs scripts/script-documentation-contract.test.mjs +``` + +`check` 会验证范围映射、文件名、重复版本、跨 schema 引用、跨库控制语句，以及可选的受保护文件清单。它不连接数据库，也不证明 SQL 语义正确。 + +## 生成测试候选 + +### 1. 只选择本次要发布的服务 + +```powershell +node deploy/sql/scripts/db-migrations.mjs plan ` + --services auth,iam ` + --regions CN ` + --out .codex-tmp-release-manifest.json +``` + +`plan` 自动生成逐库目标、完整迁移文件列表和 SHA-256。它只包含选中服务及评审确认的 schema；同一提交中其他服务的 SQL 不会被自动带入。没有迁移文件的允许 schema 目标为 `0.0.0`，范围外 schema 不生成目标。 + +生成的 manifest 是制品，不是开发者长期维护的源码文件。不要手改、提交或在测试与生产之间重新生成；测试通过后应复用同一个 manifest、SQL 包和镜像 digest。 + +### 2. 查看只读执行计划 + +```powershell +node deploy/sql/scripts/db-migrations.mjs run ` + --manifest .codex-tmp-release-manifest.json ` + --dry-run +``` + +当前阶段 `run` 只支持 `--dry-run`，用于确认要执行哪些库和版本。真实 Flyway 执行、测试数据库凭据、环境锁和 Jenkins 阶段属于后续阶段 C，尚未由本命令启用。不要自行设置开关连接真实数据库。 + +## 测试环境的目标流程 + +完成基线登记和阶段 C 接入后，测试部署按以下顺序自动完成： + +1. 构建并冻结代码、SQL manifest 和全部目标镜像。 +2. 获取测试环境锁，核验实例身份、基线和当前 Flyway 历史。 +3. 对允许迁移的 schema 做全量预检；任何库失败都不开始其他库的写入。 +4. Flyway 按完整版本链执行待迁移版本，成功版本自动跳过。 +5. 严格复验历史、结构和必要种子。 +6. 只有数据库阶段全部通过，才部署同一批应用镜像。 +7. 同一候选重复执行时只执行数据库尚未成功的版本，不重复执行成功 SQL。 + +测试环境验证至少包括空库到目标、上一发布版本到目标、带合成数据升级、重复部署、失败阻断应用部署和跨区域部分成功。测试报告必须绑定 manifest、SQL 哈希、镜像 digest 和测试起点。 + +## 生产晋级 + +生产不重新生成 SQL，也不重新编号。只选择已经通过测试的原候选制品，执行器只处理生产当前缺少的成功迁移；已成功的版本自动跳过。生产数据库高于候选目标、存在未纳入清单的成功迁移、失败记录、校验和变化或起点与测试演练不一致时，发布直接阻断。 + +生产迁移成功后才部署同批镜像。应用回滚不执行旧 SQL；只有经过兼容性记录证明的旧镜像才允许回滚。数据库逆向回滚依赖受控备份恢复或新的向前修复迁移。 + +## 失败时怎么处理 + +| 情况 | 处理方式 | +|---|---| +| `check` 失败 | 修正范围、文件名或 SQL 引用后重新检查；不能用忽略参数跳过 | +| 测试迁移失败、从未成功且未进生产 | 按方案 13.2.1 撤销旧候选、确认无残留和排队任务后，允许修订同一编号；旧制品只读保留，新候选重新完成空库、升级和数据测试 | +| 任一环境已成功执行该版本 | 不修改原文件、不复用编号修订；创建新的向前修复迁移 | +| 生产迁移失败或部分 DDL 已提交 | 停止应用部署，保留失败历史和证据，进入专项恢复；不能简单加版本号掩盖状态 | +| 国内成功、海外失败 | 保留国内成功结果，停止整批应用部署，先按区域恢复或补齐证据；不重放国内成功版本 | +| Jenkins 超时、取消或重启 | 先确认数据库会话和真实历史，状态未确认前禁止重试或清理制品 | + +## 日常维护规则 + +### 代码和目录 + +- `migration-scopes.json` 是区域、schema、服务归属和顺序的低频配置；增加服务、改数据库归属或调整依赖时才修改，并由两名开发者一起评审。 +- `migrations/` 是上线迁移唯一事实来源；不要把模块 `resources/db/`、历史脚本或临时 SQL 当作发布目录。 +- 版本号只在同一 schema 内递增，废弃编号不重新分配给其他业务目的；不同 schema 可以使用相同编号。 +- 成功迁移、冻结候选、发布 manifest、失败原文和修订记录都必须长期留档；普通构建清理不能删除它们。 +- 不提交生成的 manifest、日志、凭据、连接串或真实数据；Jenkins 制品保存 SQL、manifest 和镜像 digest。 + +### 什么时候需要新增迁移 + +新增或修改表、列、索引、约束、默认值、触发器、必要系统种子，以及会改变数据库读写要求的 Mapper/持久化 SQL，都要在评审中确认是否需要迁移。需要时用 `new` 创建；不需要时记录具体理由并补受影响数据库读写测试，不能只因为没有 SQL 文件就默认通过。 + +### 谁负责什么 + +变更开发者负责 SQL 语义、前后断言、数据影响和代码关联；另一名开发者检查范围、兼容性和测试证据；发布执行人只选择已通过测试的候选并触发自动流程。任何人都不应直接登录服务器手工执行生产 SQL。 + +## 当前实现边界 + +当前测试分支已经可以使用 `new`、`check`、`plan` 和 `run --dry-run`，并有对应离线合同测试。以下事项仍未启用： + +- 生产或测试数据库基线登记； +- Flyway 实际执行和历史表校验； +- Jenkins 的数据库阶段、环境锁和构建/部署解耦； +- 真实测试环境部署与生产晋级。 + +因此现在可以开始编写和评审迁移文件，但不能把离线检查通过当作测试或生产发布通过。 diff --git a/finance/src/test/java/com/baiyue/finance/FinanceSqlCommentContractTest.java b/finance/src/test/java/com/baiyue/finance/FinanceSqlCommentContractTest.java index 30795d6ec..20609d980 100644 --- a/finance/src/test/java/com/baiyue/finance/FinanceSqlCommentContractTest.java +++ b/finance/src/test/java/com/baiyue/finance/FinanceSqlCommentContractTest.java @@ -44,7 +44,7 @@ class FinanceSqlCommentContractTest { } } assertThat(tableCount).isEqualTo(23); - assertThat(columnCount).isEqualTo(353); + assertThat(columnCount).isEqualTo(352); assertThat(missing).as("缺少中文业务注释的Finance字段").isEmpty(); } diff --git a/scripts/README.md b/scripts/README.md index a25482f85..a24838389 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -136,6 +136,8 @@ Client 全量、Client 入库测量与计费、待审核订单和批量待分柜 部署目录的入口另有：`deploy/scripts/deploy-built-service.sh`（单服务健康检查与失败回滚）、`deploy/scripts/deploy-development-service.sh`（经 SSH 部署开发服务）、`deploy/scripts/deploy-services.sh`（按服务列表编排部署）、`deploy/scripts/initialize-test-databases.sh`（初始化测试基础设施数据库）和 `deploy/rocketmq/render-broker-config.sh`（容器内渲染并启动 RocketMQ broker）。这些脚本都会改变运行环境，不能作为普通本地检查执行。 +数据库迁移离线入口为 `deploy/sql/scripts/db-migrations.mjs`：`new` 自动生成单 schema 的下一个版本草稿，`check` 做范围/文件/跨 schema 静态检查，`plan` 生成选定服务的不可变清单，`run --dry-run` 只输出执行计划。当前不连接数据库；真实 Flyway 执行须等迁移阶段完成并获得对应环境授权。用途、参数和边界见 `deploy/sql/README.md`。 + 服务局部分析入口为 `firstleg/scripts/analyze-firstleg-dependencies.mjs` 与 `lastleg-service/scripts/analyze-lastleg-dependencies.mjs`，相邻的 `*.test.mjs` 负责对基线和违规依赖做离线回归；`warehouse/scripts/warehouse-public-master-data-contract.test.mjs` 只读校验 Warehouse 公共主数据 CSV 与 SQL。SQL 辅助文件 `scripts/db/incremental.sql` 不是当前统一部署入口，正式数据库变更以 `deploy/sql/README.md` 指向的统一增量为准。 新增可直接运行的脚本时，必须同时在本节登记用途、目标环境、是否写数据和密钥边界，并运行 `node --test scripts/script-documentation-contract.test.mjs`。仅新增 `*.test.mjs` 或 `admin-test-env-modules/` 内部模块时，分别更新对应合同范围或模块 README，不把内部文件误写成独立入口。 diff --git a/scripts/script-documentation-contract.test.mjs b/scripts/script-documentation-contract.test.mjs index 1d3c00bab..33b66b68b 100644 --- a/scripts/script-documentation-contract.test.mjs +++ b/scripts/script-documentation-contract.test.mjs @@ -14,6 +14,7 @@ const excludedDirectoryNames = new Set([ const allowedDirectoryPrefixes = [ 'scripts/', 'deploy/scripts/', + 'deploy/sql/scripts/', 'deploy/rocketmq/', 'firstleg/scripts/', 'lastleg-service/scripts/', @@ -62,6 +63,7 @@ test('all directly runnable scripts are documented in scripts/README.md', async ...await executableFiles('scripts', ['.mjs', '.ps1', '.groovy']), ...await executableFiles('scripts/openapi-validation', ['.mjs']), ...await executableFiles('deploy/scripts', ['.sh']), + ...await executableFiles('deploy/sql/scripts', ['.mjs']), ...await executableFiles('deploy/rocketmq', ['.sh']), ...await executableFiles('firstleg/scripts', ['.mjs']), ...await executableFiles('lastleg-service/scripts', ['.mjs']),