优化
This commit is contained in:
@@ -60,6 +60,30 @@
|
||||
- 模块 `src/main/resources/db/` 用作结构来源和开发追溯，不是区域部署前置；不再向历史迁移文件追加发布步骤。修改当前字段时同步模块有效建表定义及其合同。
|
||||
- 不在 SQL 中写入真实客户数据、环境地址、固定密码或 Token。
|
||||
|
||||
## 后续迁移工具（已开始实施）
|
||||
|
||||
机制启用前，以上两份统一 SQL 仍是空库初始化的唯一入口；本节工具不会修改已有数据库，也不会把当前可丢弃的开发/测试存量转换成迁移链。
|
||||
|
||||
迁移范围映射位于 `migration-scopes.json`，按区域、schema、所属服务和顺序登记。迁移文件位于 `migrations/{cn|overseas}/<schema>/`，一个文件只允许修改一个 schema。目录中的 `.gitkeep` 仅用于保留空目录，不是迁移文件。
|
||||
|
||||
完整操作请先读 [数据库迁移操作手册](数据库迁移操作手册.md)。离线入口为 `scripts/db-migrations.mjs`：
|
||||
|
||||
```powershell
|
||||
# 自动取得该 schema 的下一个整数序号并创建草稿；不连接数据库
|
||||
node deploy/sql/scripts/db-migrations.mjs new --region CN --schema baiyue_auth --description add_login_audit
|
||||
|
||||
# 校验范围、文件名、跨 schema 引用及受保护文件；不连接数据库
|
||||
node deploy/sql/scripts/db-migrations.mjs check
|
||||
|
||||
# 由选中服务自动生成逐库目标和 SQL 哈希；不手填发布 JSON
|
||||
node deploy/sql/scripts/db-migrations.mjs plan --services auth,iam --regions CN --out release-manifest.json
|
||||
|
||||
# 当前只提供安全的只读计划；实际 Flyway 执行需后续阶段显式配置并授权
|
||||
node deploy/sql/scripts/db-migrations.mjs run --manifest release-manifest.json --dry-run
|
||||
```
|
||||
|
||||
`new` 使用独占创建避免并发覆盖；成功历史或已冻结制品不能由普通命令改写。`plan` 只打包选中服务及评审确认的 schema，不会因为同一提交存在其他区域或服务 SQL 就自动扩大范围。`run` 的真实执行还未接入测试数据库和 Jenkins，当前未授权任何部署或 SQL 操作。
|
||||
|
||||
## 本地验证
|
||||
|
||||
离线合同不连接数据库：
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"services": [
|
||||
"gateway",
|
||||
"gateway-overseas",
|
||||
"auth",
|
||||
"iam",
|
||||
"order",
|
||||
"warehouse",
|
||||
"firstleg",
|
||||
"file",
|
||||
"audit-service",
|
||||
"event-bridge-service",
|
||||
"finance",
|
||||
"integration-service",
|
||||
"print",
|
||||
"lastleg-service",
|
||||
"warehouse-overseas-service"
|
||||
],
|
||||
"regions": {
|
||||
"CN": {
|
||||
"baseline": "baseline/cn/cn-baseline.sql",
|
||||
"schemas": [
|
||||
{ "name": "baiyue_auth", "services": ["auth"], "order": 10 },
|
||||
{ "name": "baiyue_iam", "services": ["iam"], "order": 20 },
|
||||
{ "name": "baiyue_order", "services": ["order"], "order": 30 },
|
||||
{ "name": "baiyue_warehouse", "services": ["warehouse"], "order": 40 },
|
||||
{ "name": "baiyue_firstleg", "services": ["firstleg"], "order": 50 },
|
||||
{ "name": "baiyue_file", "services": ["file"], "order": 60 },
|
||||
{ "name": "baiyue_audit", "services": ["audit-service"], "order": 70 },
|
||||
{ "name": "baiyue_event_bridge", "services": ["event-bridge-service"], "order": 80 },
|
||||
{ "name": "baiyue_finance", "services": ["finance"], "order": 90 }
|
||||
]
|
||||
},
|
||||
"OVERSEAS": {
|
||||
"baseline": "baseline/overseas/overseas-baseline.sql",
|
||||
"schemas": [
|
||||
{ "name": "baiyue_lastleg", "services": ["lastleg-service"], "order": 10 },
|
||||
{ "name": "baiyue_lastlag_warehouse", "services": ["warehouse-overseas-service"], "order": 20 },
|
||||
{ "name": "baiyue_event_bridge_overseas", "services": ["event-bridge-service"], "order": 30 }
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,322 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { createHash } from 'node:crypto'
|
||||
import { mkdir, open, readFile, readdir, stat, writeFile } from 'node:fs/promises'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import path from 'node:path'
|
||||
import process from 'node:process'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url))
|
||||
const SQL_ROOT = path.resolve(SCRIPT_DIR, '..')
|
||||
const SCOPES_PATH = path.join(SQL_ROOT, 'migration-scopes.json')
|
||||
const MIGRATION_NAME = /^V(\d+)__([a-z0-9][a-z0-9_-]*)\.sql$/
|
||||
|
||||
function fail(message) {
|
||||
throw new Error(message)
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const [command = 'help', ...rest] = argv
|
||||
const args = { command, _: [] }
|
||||
for (let index = 0; index < rest.length; index += 1) {
|
||||
const item = rest[index]
|
||||
if (!item.startsWith('--')) {
|
||||
args._.push(item)
|
||||
continue
|
||||
}
|
||||
const equals = item.indexOf('=')
|
||||
if (equals >= 0) {
|
||||
args[item.slice(2, equals)] = item.slice(equals + 1)
|
||||
continue
|
||||
}
|
||||
const key = item.slice(2)
|
||||
const next = rest[index + 1]
|
||||
if (next && !next.startsWith('--')) {
|
||||
args[key] = next
|
||||
index += 1
|
||||
} else {
|
||||
args[key] = true
|
||||
}
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
function absoluteFrom(root, value) {
|
||||
return path.isAbsolute(value) ? value : path.resolve(root, value)
|
||||
}
|
||||
|
||||
async function loadScopes(scopesPath = SCOPES_PATH) {
|
||||
const scopes = JSON.parse(await readFile(scopesPath, 'utf8'))
|
||||
if (scopes.schemaVersion !== 1 || !scopes.regions || typeof scopes.regions !== 'object') {
|
||||
fail('migration-scopes.json 必须使用 schemaVersion=1 并提供 regions')
|
||||
}
|
||||
const seenSchemas = new Set()
|
||||
for (const [region, config] of Object.entries(scopes.regions)) {
|
||||
if (!/^[A-Z][A-Z0-9_]*$/.test(region)) fail(`区域名非法：${region}`)
|
||||
if (!Array.isArray(config.schemas) || config.schemas.length === 0) fail(`${region} 没有 schema 清单`)
|
||||
const orders = new Set()
|
||||
for (const schema of config.schemas) {
|
||||
if (!/^[a-z][a-z0-9_]*$/.test(schema.name)) fail(`schema 名称非法：${schema.name}`)
|
||||
if (seenSchemas.has(schema.name)) fail(`schema 重复登记：${schema.name}`)
|
||||
if (!Number.isInteger(schema.order) || schema.order < 1 || orders.has(schema.order)) fail(`${region} 的 schema order 非法或重复：${schema.name}`)
|
||||
if (!Array.isArray(schema.services) || schema.services.length === 0) fail(`${schema.name} 没有所属服务`)
|
||||
seenSchemas.add(schema.name)
|
||||
orders.add(schema.order)
|
||||
}
|
||||
}
|
||||
return scopes
|
||||
}
|
||||
|
||||
function regionConfig(scopes, region) {
|
||||
const value = scopes.regions[region]
|
||||
if (!value) fail(`未知区域：${region}；允许值：${Object.keys(scopes.regions).join(', ')}`)
|
||||
return value
|
||||
}
|
||||
|
||||
function schemaConfig(scopes, region, schemaName) {
|
||||
const config = regionConfig(scopes, region)
|
||||
const schema = config.schemas.find((item) => item.name === schemaName)
|
||||
if (!schema) fail(`${region} 未登记 schema：${schemaName}`)
|
||||
return schema
|
||||
}
|
||||
|
||||
async function listMigrations(sqlRoot, region, schemaName) {
|
||||
const directory = path.join(sqlRoot, 'migrations', region.toLowerCase(), schemaName)
|
||||
let entries = []
|
||||
try {
|
||||
entries = await readdir(directory, { withFileTypes: true })
|
||||
} catch (error) {
|
||||
if (error.code === 'ENOENT') return []
|
||||
throw error
|
||||
}
|
||||
const migrations = []
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile() || entry.name.startsWith('.')) continue
|
||||
const match = entry.name.match(MIGRATION_NAME)
|
||||
if (!match) fail(`${path.relative(sqlRoot, path.join(directory, entry.name))} 文件名不符合 V<integer>__description.sql`)
|
||||
const version = Number(match[1])
|
||||
if (!Number.isSafeInteger(version) || version < 1) fail(`迁移版本非法：${entry.name}`)
|
||||
migrations.push({
|
||||
version,
|
||||
description: match[2],
|
||||
fileName: entry.name,
|
||||
relativePath: path.relative(sqlRoot, path.join(directory, entry.name)).replaceAll(path.sep, '/'),
|
||||
absolutePath: path.join(directory, entry.name),
|
||||
})
|
||||
}
|
||||
migrations.sort((left, right) => left.version - right.version || left.fileName.localeCompare(right.fileName))
|
||||
for (let index = 1; index < migrations.length; index += 1) {
|
||||
if (migrations[index - 1].version === migrations[index].version) fail(`${region}/${schemaName} 版本重复：${migrations[index].version}`)
|
||||
}
|
||||
return migrations
|
||||
}
|
||||
|
||||
async function sha256(filePath) {
|
||||
return createHash('sha256').update(await readFile(filePath)).digest('hex')
|
||||
}
|
||||
|
||||
function selectedServices(value) {
|
||||
if (!value || value === 'ALL') return null
|
||||
const services = value.split(',').map((item) => item.trim()).filter(Boolean)
|
||||
if (services.length === 0) fail('--services 不能为空')
|
||||
return [...new Set(services)]
|
||||
}
|
||||
|
||||
function selectSchemas(scopes, region, services) {
|
||||
const config = regionConfig(scopes, region)
|
||||
const selected = services === null
|
||||
? config.schemas
|
||||
: config.schemas.filter((schema) => schema.services.some((service) => services.includes(service)))
|
||||
return [...selected].sort((left, right) => left.order - right.order)
|
||||
}
|
||||
|
||||
async function buildPlan({ sqlRoot = SQL_ROOT, scopes, regions, services, commit = process.env.GIT_COMMIT ?? null }) {
|
||||
const knownServices = new Set(scopes.services ?? Object.values(scopes.regions).flatMap((config) => config.schemas.flatMap((schema) => schema.services)))
|
||||
const unknownServices = (services ?? []).filter((service) => !knownServices.has(service))
|
||||
if (unknownServices.length) fail(`范围映射中未登记服务：${unknownServices.join(', ')}`)
|
||||
const plan = {
|
||||
schemaVersion: 1,
|
||||
sourceCommit: commit,
|
||||
services: services ?? ['ALL'],
|
||||
regions: [],
|
||||
}
|
||||
for (const region of regions) {
|
||||
const config = regionConfig(scopes, region)
|
||||
const schemas = []
|
||||
for (const schema of selectSchemas(scopes, region, services)) {
|
||||
const migrations = await listMigrations(sqlRoot, region, schema.name)
|
||||
schemas.push({
|
||||
schema: schema.name,
|
||||
services: schema.services,
|
||||
order: schema.order,
|
||||
migrationDirectory: `migrations/${region.toLowerCase()}/${schema.name}`,
|
||||
baseline: config.baseline,
|
||||
target: migrations.length ? String(migrations.at(-1).version) : '0.0.0',
|
||||
migrations: await Promise.all(migrations.map(async (migration) => ({
|
||||
version: String(migration.version),
|
||||
description: migration.description,
|
||||
path: migration.relativePath,
|
||||
sha256: await sha256(migration.absolutePath),
|
||||
}))),
|
||||
})
|
||||
}
|
||||
plan.regions.push({ region, schemas })
|
||||
}
|
||||
return plan
|
||||
}
|
||||
|
||||
async function createMigration({ sqlRoot = SQL_ROOT, scopes, region, schemaName, description }) {
|
||||
const schema = schemaConfig(scopes, region, schemaName)
|
||||
const directory = path.join(sqlRoot, 'migrations', region.toLowerCase(), schema.name)
|
||||
await mkdir(directory, { recursive: true })
|
||||
const normalizedDescription = description.trim()
|
||||
if (!normalizedDescription) fail('必须填写变更说明')
|
||||
const safeDescription = normalizedDescription.toLowerCase().replaceAll(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, '') || 'change'
|
||||
for (let attempt = 0; attempt < 20; attempt += 1) {
|
||||
const migrations = await listMigrations(sqlRoot, region, schema.name)
|
||||
const version = (migrations.at(-1)?.version ?? 0) + 1
|
||||
const fileName = `V${version}__${safeDescription}.sql`
|
||||
const filePath = path.join(directory, fileName)
|
||||
try {
|
||||
const handle = await open(filePath, 'wx')
|
||||
await handle.writeFile([
|
||||
`-- Region: ${region}`,
|
||||
`-- Schema: ${schema.name}`,
|
||||
`-- Purpose: ${description.trim()}`,
|
||||
'-- Preconditions:',
|
||||
'-- Postconditions:',
|
||||
'',
|
||||
].join('\n'), 'utf8')
|
||||
await handle.close()
|
||||
return { region, schema: schema.name, version, fileName, path: path.relative(sqlRoot, filePath).replaceAll(path.sep, '/') }
|
||||
} catch (error) {
|
||||
if (error.code !== 'EEXIST') throw error
|
||||
}
|
||||
}
|
||||
fail(`${region}/${schema.name} 并发生成迁移失败：重试次数已耗尽`)
|
||||
}
|
||||
|
||||
async function checkProtected(sqlRoot, protectedPath) {
|
||||
if (!protectedPath) return []
|
||||
const document = JSON.parse(await readFile(protectedPath, 'utf8'))
|
||||
const entries = document.files ?? document.migrations ?? []
|
||||
if (!Array.isArray(entries)) fail('保护清单的 files/migrations 必须是数组')
|
||||
const errors = []
|
||||
for (const entry of entries) {
|
||||
if (!['success', 'frozen'].includes(entry.status)) continue
|
||||
if (!entry.path || !/^[a-f0-9]{64}$/.test(entry.sha256 ?? '')) fail('保护清单条目必须包含 path 和 64 位 sha256')
|
||||
const current = await sha256(absoluteFrom(sqlRoot, entry.path))
|
||||
if (current !== entry.sha256) errors.push(`受保护迁移已修改：${entry.path}`)
|
||||
}
|
||||
return errors
|
||||
}
|
||||
|
||||
async function checkSqlFiles(sqlRoot, scopes, regions) {
|
||||
const errors = []
|
||||
const schemasByName = new Map()
|
||||
for (const region of regions) {
|
||||
for (const schema of regionConfig(scopes, region).schemas) schemasByName.set(schema.name, { region, schema })
|
||||
}
|
||||
for (const [schemaName, { region }] of schemasByName) {
|
||||
const migrations = await listMigrations(sqlRoot, region, schemaName)
|
||||
for (const migration of migrations) {
|
||||
const sql = await readFile(migration.absolutePath, 'utf8')
|
||||
for (const candidate of schemasByName.keys()) {
|
||||
if (candidate !== schemaName && new RegExp(`(?:\\b|\`)${candidate}(?:\\b|\`)`, 'i').test(sql)) {
|
||||
errors.push(`${migration.relativePath} 引用了其他 schema：${candidate}`)
|
||||
}
|
||||
}
|
||||
if (/\b(?:DROP\s+DATABASE|CREATE\s+DATABASE|USE\s+)/i.test(sql)) errors.push(`${migration.relativePath} 不得包含跨库控制语句`)
|
||||
}
|
||||
}
|
||||
return errors
|
||||
}
|
||||
|
||||
async function check({ sqlRoot = SQL_ROOT, scopes, regions = Object.keys(scopes.regions), protectedPath }) {
|
||||
const errors = await checkSqlFiles(sqlRoot, scopes, regions)
|
||||
errors.push(...await checkProtected(sqlRoot, protectedPath))
|
||||
if (errors.length) fail(errors.join('\n'))
|
||||
return { ok: true, regions, checkedAt: new Date().toISOString() }
|
||||
}
|
||||
|
||||
async function runManifest({ sqlRoot = SQL_ROOT, manifestPath, dryRun = false }) {
|
||||
const manifest = JSON.parse(await readFile(manifestPath, 'utf8'))
|
||||
if (manifest.schemaVersion !== 1 || !Array.isArray(manifest.regions)) fail('发布清单格式不受支持')
|
||||
const commands = []
|
||||
for (const region of manifest.regions) {
|
||||
for (const schema of region.schemas ?? []) {
|
||||
if (schema.target === '0.0.0' && (schema.migrations ?? []).length === 0) continue
|
||||
const location = path.join(sqlRoot, schema.migrationDirectory)
|
||||
await stat(location).catch(() => fail(`迁移目录不存在：${schema.migrationDirectory}`))
|
||||
commands.push({ region: region.region, schema: schema.schema, target: schema.target, location })
|
||||
}
|
||||
}
|
||||
if (dryRun) return { mode: 'dry-run', commands }
|
||||
if (process.env.DB_MIGRATION_ALLOW !== '1') fail('实际数据库执行被保护：先设置 DB_MIGRATION_ALLOW=1，并显式提供测试环境凭据')
|
||||
if (!process.env.FLYWAY_BIN) fail('实际数据库执行需要 FLYWAY_BIN，阶段 B 只支持 --dry-run')
|
||||
const results = []
|
||||
for (const command of commands) {
|
||||
const result = spawnSync(process.env.FLYWAY_BIN, [
|
||||
'-locations=filesystem:' + command.location,
|
||||
'-schemas=' + command.schema,
|
||||
'-defaultSchema=' + command.schema,
|
||||
'-table=flyway_schema_history',
|
||||
'-baselineOnMigrate=false',
|
||||
'-cleanDisabled=true',
|
||||
'-outOfOrder=false',
|
||||
'-validateOnMigrate=true',
|
||||
'-target=' + command.target,
|
||||
'migrate',
|
||||
], { stdio: 'inherit', env: process.env })
|
||||
if (result.status !== 0) fail(`${command.region}/${command.schema} Flyway 执行失败，退出码 ${result.status}`)
|
||||
results.push(command)
|
||||
}
|
||||
return { mode: 'execute', commands: results }
|
||||
}
|
||||
|
||||
function usage() {
|
||||
console.log(`用法：\n node deploy/sql/scripts/db-migrations.mjs new --region CN --schema baiyue_auth --description add_login_audit\n node deploy/sql/scripts/db-migrations.mjs check [--protected <manifest>]\n node deploy/sql/scripts/db-migrations.mjs plan --services auth,iam --regions CN,OVERSEAS --out <file>\n node deploy/sql/scripts/db-migrations.mjs run --manifest <file> --dry-run`)
|
||||
}
|
||||
|
||||
async function main(argv = process.argv.slice(2)) {
|
||||
const args = parseArgs(argv)
|
||||
const scopesPath = args.scopes ? absoluteFrom(process.cwd(), args.scopes) : SCOPES_PATH
|
||||
const scopes = await loadScopes(scopesPath)
|
||||
const sqlRoot = path.dirname(scopesPath)
|
||||
if (args.command === 'help') return usage()
|
||||
if (args.command === 'new') {
|
||||
const result = await createMigration({ sqlRoot, scopes, region: String(args.region ?? ''), schemaName: String(args.schema ?? ''), description: String(args.description ?? '') })
|
||||
console.log(JSON.stringify(result, null, 2))
|
||||
return
|
||||
}
|
||||
if (args.command === 'check') {
|
||||
const result = await check({ sqlRoot, scopes, protectedPath: args.protected ? absoluteFrom(process.cwd(), args.protected) : undefined })
|
||||
console.log(JSON.stringify(result, null, 2))
|
||||
return
|
||||
}
|
||||
if (args.command === 'plan') {
|
||||
const regions = String(args.regions ?? Object.keys(scopes.regions).join(',')).split(',').map((item) => item.trim()).filter(Boolean)
|
||||
const plan = await buildPlan({ sqlRoot, scopes, regions, services: selectedServices(args.services) })
|
||||
const output = JSON.stringify(plan, null, 2) + '\n'
|
||||
if (args.out) await writeFile(absoluteFrom(process.cwd(), args.out), output, 'utf8')
|
||||
else process.stdout.write(output)
|
||||
return
|
||||
}
|
||||
if (args.command === 'run') {
|
||||
const result = await runManifest({ sqlRoot, manifestPath: absoluteFrom(process.cwd(), String(args.manifest ?? '')), dryRun: Boolean(args['dry-run']) })
|
||||
console.log(JSON.stringify(result, null, 2))
|
||||
return
|
||||
}
|
||||
usage()
|
||||
fail(`未知命令：${args.command}`)
|
||||
}
|
||||
|
||||
export { buildPlan, check, createMigration, listMigrations as discoverMigrations, loadScopes, parseArgs, runManifest }
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
main().catch((error) => {
|
||||
console.error(`ERROR: ${error.message}`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import test from 'node:test'
|
||||
import {
|
||||
buildPlan,
|
||||
check,
|
||||
createMigration,
|
||||
discoverMigrations,
|
||||
loadScopes,
|
||||
runManifest,
|
||||
} from './db-migrations.mjs'
|
||||
|
||||
async function fixture() {
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), 'db-migrations-'))
|
||||
const sqlRoot = path.join(root, 'sql')
|
||||
await mkdir(path.join(sqlRoot, 'migrations', 'cn', 'baiyue_auth'), { recursive: true })
|
||||
await writeFile(path.join(sqlRoot, 'scopes.json'), JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
regions: {
|
||||
CN: {
|
||||
baseline: 'baseline/cn.sql',
|
||||
schemas: [
|
||||
{ name: 'baiyue_auth', services: ['auth'], order: 10 },
|
||||
{ name: 'baiyue_order', services: ['order'], order: 20 },
|
||||
],
|
||||
},
|
||||
},
|
||||
}), 'utf8')
|
||||
return { root, sqlRoot, scopesPath: path.join(sqlRoot, 'scopes.json') }
|
||||
}
|
||||
|
||||
test('new 自动分配 schema 内下一个版本，并且可重复执行检查', async (t) => {
|
||||
const context = await fixture()
|
||||
t.after(() => rm(context.root, { recursive: true, force: true }))
|
||||
const scopes = await loadScopes(context.scopesPath)
|
||||
const first = await createMigration({ sqlRoot: context.sqlRoot, scopes, region: 'CN', schemaName: 'baiyue_auth', description: 'Add login audit' })
|
||||
assert.equal(first.fileName, 'V1__add_login_audit.sql')
|
||||
const second = await createMigration({ sqlRoot: context.sqlRoot, scopes, region: 'CN', schemaName: 'baiyue_auth', description: 'add_session_index' })
|
||||
assert.equal(second.fileName, 'V2__add_session_index.sql')
|
||||
assert.match(await readFile(path.join(context.sqlRoot, first.path), 'utf8'), /Schema: baiyue_auth/)
|
||||
assert.equal((await discoverMigrations(context.sqlRoot, 'CN', 'baiyue_auth')).length, 2)
|
||||
})
|
||||
|
||||
test('plan 只包含选中服务，不把同提交其他 schema 的 SQL 带入制品', async (t) => {
|
||||
const context = await fixture()
|
||||
t.after(() => rm(context.root, { recursive: true, force: true }))
|
||||
const scopes = await loadScopes(context.scopesPath)
|
||||
const created = await createMigration({ sqlRoot: context.sqlRoot, scopes, region: 'CN', schemaName: 'baiyue_auth', description: 'add_login_audit' })
|
||||
const plan = await buildPlan({ sqlRoot: context.sqlRoot, scopes, regions: ['CN'], services: ['auth'], commit: 'abc1234' })
|
||||
assert.equal(plan.sourceCommit, 'abc1234')
|
||||
assert.deepEqual(plan.regions[0].schemas.map((schema) => schema.schema), ['baiyue_auth'])
|
||||
assert.equal(plan.regions[0].schemas[0].target, '1')
|
||||
assert.equal(plan.regions[0].schemas[0].migrations[0].path, created.path)
|
||||
})
|
||||
|
||||
test('check 拒绝跨 schema 引用和跨库控制语句', async (t) => {
|
||||
const context = await fixture()
|
||||
t.after(() => rm(context.root, { recursive: true, force: true }))
|
||||
const scopes = await loadScopes(context.scopesPath)
|
||||
await writeFile(path.join(context.sqlRoot, 'migrations', 'cn', 'baiyue_auth', 'V1__bad.sql'), 'ALTER TABLE baiyue_order.t ADD c INT;\nUSE baiyue_order;\n', 'utf8')
|
||||
await assert.rejects(() => check({ sqlRoot: context.sqlRoot, scopes, regions: ['CN'] }), /其他 schema|跨库控制语句/)
|
||||
})
|
||||
|
||||
test('run 默认只能 dry-run，不能无意连接数据库', async (t) => {
|
||||
const context = await fixture()
|
||||
t.after(() => rm(context.root, { recursive: true, force: true }))
|
||||
const scopes = await loadScopes(context.scopesPath)
|
||||
await createMigration({ sqlRoot: context.sqlRoot, scopes, region: 'CN', schemaName: 'baiyue_auth', description: 'add_login_audit' })
|
||||
const plan = await buildPlan({ sqlRoot: context.sqlRoot, scopes, regions: ['CN'], services: ['auth'] })
|
||||
const manifestPath = path.join(context.root, 'release-manifest.json')
|
||||
await writeFile(manifestPath, JSON.stringify(plan), 'utf8')
|
||||
const dryRun = await runManifest({ sqlRoot: context.sqlRoot, manifestPath, dryRun: true })
|
||||
assert.equal(dryRun.mode, 'dry-run')
|
||||
await assert.rejects(() => runManifest({ sqlRoot: context.sqlRoot, manifestPath }), /DB_MIGRATION_ALLOW/)
|
||||
})
|
||||
@@ -0,0 +1,141 @@
|
||||
# 数据库迁移操作手册
|
||||
|
||||
本文是开发、测试和发布人员的日常操作入口。设计原则和生产边界见仓库文档 [数据库版本迁移与 Jenkins 自动发布方案](../../../docs/数据库版本迁移与Jenkins自动发布方案.md)；当前阶段的初始化事实见本目录 [README.md](README.md)。
|
||||
|
||||
## 先分清两类 SQL
|
||||
|
||||
| 类型 | 位置 | 用途 | 能否在已有业务库重复执行 |
|
||||
|---|---|---|---|
|
||||
| 空库初始化 | `deploy/sql/头程数据库统一增量-MySQL8.sql`、`海外区域统一增量-MySQL8.sql` | 初始化一个不存在或获授权清空的区域数据库 | 不能 |
|
||||
| 版本迁移 | `deploy/sql/migrations/{cn|overseas}/<schema>/V<数字>__<说明>.sql` | 机制启用后对已有数据库逐步升级 | 由 Flyway 按历史自动跳过已成功版本 |
|
||||
|
||||
当前测试分支只完成了版本迁移的离线工具和目录骨架。没有冻结基线、没有具体业务变更时，不要复制初始化 SQL 生成 `V1`，也不要连接测试或生产数据库试跑。
|
||||
|
||||
## 日常开发：新建和修改迁移
|
||||
|
||||
以下命令均从 `baiyue-firstleg` 根目录执行，PowerShell 和 Bash 都可以使用 Node 命令。
|
||||
|
||||
### 1. 确认所属区域和 schema
|
||||
|
||||
打开 [migration-scopes.json](migration-scopes.json)，只选择已登记的区域和 schema。服务没有数据库时不创建空迁移；例如 `gateway` 不对应 schema。
|
||||
|
||||
```powershell
|
||||
node deploy/sql/scripts/db-migrations.mjs check
|
||||
```
|
||||
|
||||
### 2. 自动创建迁移文件
|
||||
|
||||
```powershell
|
||||
node deploy/sql/scripts/db-migrations.mjs new `
|
||||
--region CN `
|
||||
--schema baiyue_auth `
|
||||
--description add_login_audit
|
||||
```
|
||||
|
||||
工具会读取该 schema 当前最大编号并创建下一个文件，例如：
|
||||
|
||||
```text
|
||||
deploy/sql/migrations/cn/baiyue_auth/V1__add_login_audit.sql
|
||||
```
|
||||
|
||||
开发者只输入区域、schema 和说明，不输入版本号。说明中的英文字母、数字和下划线会用于文件名；只有中文说明时文件名使用 `change`，完整中文说明仍保留在 SQL 头部。并发创建使用独占文件创建，不覆盖已有文件。
|
||||
|
||||
### 3. 填写 SQL 和断言
|
||||
|
||||
在生成的文件中完成实际 SQL，并补充文件头的前置、后置检查。一个文件只修改一个 schema，不写 `USE`、`CREATE DATABASE`、跨 schema 限定名或真实业务数据。
|
||||
|
||||
同一文件在开发阶段可以反复编辑，但一旦在保留历史的环境成功执行或进入冻结候选，就不能再改。没有成功记录但测试候选已失败时，只有方案第 13.2.1 节的受控修订流程可以保留原编号修正。
|
||||
|
||||
### 4. 提交前离线检查
|
||||
|
||||
```powershell
|
||||
node deploy/sql/scripts/db-migrations.mjs check
|
||||
node --test deploy/sql/scripts/db-migrations.test.mjs scripts/script-documentation-contract.test.mjs
|
||||
```
|
||||
|
||||
`check` 会验证范围映射、文件名、重复版本、跨 schema 引用、跨库控制语句，以及可选的受保护文件清单。它不连接数据库，也不证明 SQL 语义正确。
|
||||
|
||||
## 生成测试候选
|
||||
|
||||
### 1. 只选择本次要发布的服务
|
||||
|
||||
```powershell
|
||||
node deploy/sql/scripts/db-migrations.mjs plan `
|
||||
--services auth,iam `
|
||||
--regions CN `
|
||||
--out .codex-tmp-release-manifest.json
|
||||
```
|
||||
|
||||
`plan` 自动生成逐库目标、完整迁移文件列表和 SHA-256。它只包含选中服务及评审确认的 schema；同一提交中其他服务的 SQL 不会被自动带入。没有迁移文件的允许 schema 目标为 `0.0.0`，范围外 schema 不生成目标。
|
||||
|
||||
生成的 manifest 是制品，不是开发者长期维护的源码文件。不要手改、提交或在测试与生产之间重新生成；测试通过后应复用同一个 manifest、SQL 包和镜像 digest。
|
||||
|
||||
### 2. 查看只读执行计划
|
||||
|
||||
```powershell
|
||||
node deploy/sql/scripts/db-migrations.mjs run `
|
||||
--manifest .codex-tmp-release-manifest.json `
|
||||
--dry-run
|
||||
```
|
||||
|
||||
当前阶段 `run` 只支持 `--dry-run`，用于确认要执行哪些库和版本。真实 Flyway 执行、测试数据库凭据、环境锁和 Jenkins 阶段属于后续阶段 C，尚未由本命令启用。不要自行设置开关连接真实数据库。
|
||||
|
||||
## 测试环境的目标流程
|
||||
|
||||
完成基线登记和阶段 C 接入后，测试部署按以下顺序自动完成：
|
||||
|
||||
1. 构建并冻结代码、SQL manifest 和全部目标镜像。
|
||||
2. 获取测试环境锁，核验实例身份、基线和当前 Flyway 历史。
|
||||
3. 对允许迁移的 schema 做全量预检；任何库失败都不开始其他库的写入。
|
||||
4. Flyway 按完整版本链执行待迁移版本，成功版本自动跳过。
|
||||
5. 严格复验历史、结构和必要种子。
|
||||
6. 只有数据库阶段全部通过，才部署同一批应用镜像。
|
||||
7. 同一候选重复执行时只执行数据库尚未成功的版本，不重复执行成功 SQL。
|
||||
|
||||
测试环境验证至少包括空库到目标、上一发布版本到目标、带合成数据升级、重复部署、失败阻断应用部署和跨区域部分成功。测试报告必须绑定 manifest、SQL 哈希、镜像 digest 和测试起点。
|
||||
|
||||
## 生产晋级
|
||||
|
||||
生产不重新生成 SQL，也不重新编号。只选择已经通过测试的原候选制品，执行器只处理生产当前缺少的成功迁移；已成功的版本自动跳过。生产数据库高于候选目标、存在未纳入清单的成功迁移、失败记录、校验和变化或起点与测试演练不一致时，发布直接阻断。
|
||||
|
||||
生产迁移成功后才部署同批镜像。应用回滚不执行旧 SQL；只有经过兼容性记录证明的旧镜像才允许回滚。数据库逆向回滚依赖受控备份恢复或新的向前修复迁移。
|
||||
|
||||
## 失败时怎么处理
|
||||
|
||||
| 情况 | 处理方式 |
|
||||
|---|---|
|
||||
| `check` 失败 | 修正范围、文件名或 SQL 引用后重新检查；不能用忽略参数跳过 |
|
||||
| 测试迁移失败、从未成功且未进生产 | 按方案 13.2.1 撤销旧候选、确认无残留和排队任务后，允许修订同一编号；旧制品只读保留，新候选重新完成空库、升级和数据测试 |
|
||||
| 任一环境已成功执行该版本 | 不修改原文件、不复用编号修订；创建新的向前修复迁移 |
|
||||
| 生产迁移失败或部分 DDL 已提交 | 停止应用部署，保留失败历史和证据，进入专项恢复；不能简单加版本号掩盖状态 |
|
||||
| 国内成功、海外失败 | 保留国内成功结果，停止整批应用部署，先按区域恢复或补齐证据；不重放国内成功版本 |
|
||||
| Jenkins 超时、取消或重启 | 先确认数据库会话和真实历史，状态未确认前禁止重试或清理制品 |
|
||||
|
||||
## 日常维护规则
|
||||
|
||||
### 代码和目录
|
||||
|
||||
- `migration-scopes.json` 是区域、schema、服务归属和顺序的低频配置；增加服务、改数据库归属或调整依赖时才修改，并由两名开发者一起评审。
|
||||
- `migrations/` 是上线迁移唯一事实来源；不要把模块 `resources/db/`、历史脚本或临时 SQL 当作发布目录。
|
||||
- 版本号只在同一 schema 内递增，废弃编号不重新分配给其他业务目的；不同 schema 可以使用相同编号。
|
||||
- 成功迁移、冻结候选、发布 manifest、失败原文和修订记录都必须长期留档；普通构建清理不能删除它们。
|
||||
- 不提交生成的 manifest、日志、凭据、连接串或真实数据；Jenkins 制品保存 SQL、manifest 和镜像 digest。
|
||||
|
||||
### 什么时候需要新增迁移
|
||||
|
||||
新增或修改表、列、索引、约束、默认值、触发器、必要系统种子，以及会改变数据库读写要求的 Mapper/持久化 SQL，都要在评审中确认是否需要迁移。需要时用 `new` 创建；不需要时记录具体理由并补受影响数据库读写测试，不能只因为没有 SQL 文件就默认通过。
|
||||
|
||||
### 谁负责什么
|
||||
|
||||
变更开发者负责 SQL 语义、前后断言、数据影响和代码关联；另一名开发者检查范围、兼容性和测试证据；发布执行人只选择已通过测试的候选并触发自动流程。任何人都不应直接登录服务器手工执行生产 SQL。
|
||||
|
||||
## 当前实现边界
|
||||
|
||||
当前测试分支已经可以使用 `new`、`check`、`plan` 和 `run --dry-run`，并有对应离线合同测试。以下事项仍未启用：
|
||||
|
||||
- 生产或测试数据库基线登记；
|
||||
- Flyway 实际执行和历史表校验；
|
||||
- Jenkins 的数据库阶段、环境锁和构建/部署解耦；
|
||||
- 真实测试环境部署与生产晋级。
|
||||
|
||||
因此现在可以开始编写和评审迁移文件，但不能把离线检查通过当作测试或生产发布通过。
|
||||
@@ -44,7 +44,7 @@ class FinanceSqlCommentContractTest {
|
||||
}
|
||||
}
|
||||
assertThat(tableCount).isEqualTo(23);
|
||||
assertThat(columnCount).isEqualTo(353);
|
||||
assertThat(columnCount).isEqualTo(352);
|
||||
assertThat(missing).as("缺少中文业务注释的Finance字段").isEmpty();
|
||||
}
|
||||
|
||||
|
||||
@@ -136,6 +136,8 @@ Client 全量、Client 入库测量与计费、待审核订单和批量待分柜
|
||||
|
||||
部署目录的入口另有：`deploy/scripts/deploy-built-service.sh`（单服务健康检查与失败回滚）、`deploy/scripts/deploy-development-service.sh`（经 SSH 部署开发服务）、`deploy/scripts/deploy-services.sh`（按服务列表编排部署）、`deploy/scripts/initialize-test-databases.sh`（初始化测试基础设施数据库）和 `deploy/rocketmq/render-broker-config.sh`（容器内渲染并启动 RocketMQ broker）。这些脚本都会改变运行环境，不能作为普通本地检查执行。
|
||||
|
||||
数据库迁移离线入口为 `deploy/sql/scripts/db-migrations.mjs`：`new` 自动生成单 schema 的下一个版本草稿，`check` 做范围/文件/跨 schema 静态检查，`plan` 生成选定服务的不可变清单，`run --dry-run` 只输出执行计划。当前不连接数据库；真实 Flyway 执行须等迁移阶段完成并获得对应环境授权。用途、参数和边界见 `deploy/sql/README.md`。
|
||||
|
||||
服务局部分析入口为 `firstleg/scripts/analyze-firstleg-dependencies.mjs` 与 `lastleg-service/scripts/analyze-lastleg-dependencies.mjs`，相邻的 `*.test.mjs` 负责对基线和违规依赖做离线回归；`warehouse/scripts/warehouse-public-master-data-contract.test.mjs` 只读校验 Warehouse 公共主数据 CSV 与 SQL。SQL 辅助文件 `scripts/db/incremental.sql` 不是当前统一部署入口，正式数据库变更以 `deploy/sql/README.md` 指向的统一增量为准。
|
||||
|
||||
新增可直接运行的脚本时，必须同时在本节登记用途、目标环境、是否写数据和密钥边界，并运行 `node --test scripts/script-documentation-contract.test.mjs`。仅新增 `*.test.mjs` 或 `admin-test-env-modules/` 内部模块时，分别更新对应合同范围或模块 README，不把内部文件误写成独立入口。
|
||||
|
||||
@@ -14,6 +14,7 @@ const excludedDirectoryNames = new Set([
|
||||
const allowedDirectoryPrefixes = [
|
||||
'scripts/',
|
||||
'deploy/scripts/',
|
||||
'deploy/sql/scripts/',
|
||||
'deploy/rocketmq/',
|
||||
'firstleg/scripts/',
|
||||
'lastleg-service/scripts/',
|
||||
@@ -62,6 +63,7 @@ test('all directly runnable scripts are documented in scripts/README.md', async
|
||||
...await executableFiles('scripts', ['.mjs', '.ps1', '.groovy']),
|
||||
...await executableFiles('scripts/openapi-validation', ['.mjs']),
|
||||
...await executableFiles('deploy/scripts', ['.sh']),
|
||||
...await executableFiles('deploy/sql/scripts', ['.mjs']),
|
||||
...await executableFiles('deploy/rocketmq', ['.sh']),
|
||||
...await executableFiles('firstleg/scripts', ['.mjs']),
|
||||
...await executableFiles('lastleg-service/scripts', ['.mjs']),
|
||||
|
||||
Reference in New Issue
Block a user