155 lines
8.2 KiB
Python
155 lines
8.2 KiB
Python
"""CLI database backup, integrity and restore operations."""
|
|
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from contextlib import closing
|
|
from datetime import datetime, timedelta
|
|
from unittest.mock import patch
|
|
|
|
from logistics_chat import database_backup
|
|
from pathlib import Path
|
|
|
|
|
|
class DatabaseMaintenanceTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.directory = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.directory.cleanup)
|
|
self.root = Path(self.directory.name)
|
|
self.database = self.root/'history.db'
|
|
self.backups = self.root/'backups'
|
|
self.backups.mkdir()
|
|
with closing(sqlite3.connect(self.database)) as connection:
|
|
connection.execute('CREATE TABLE records(value TEXT NOT NULL)')
|
|
connection.execute("INSERT INTO records VALUES('original')")
|
|
connection.commit()
|
|
|
|
def run_cli(self, *args):
|
|
return subprocess.run([sys.executable, 'scripts/db_maintenance.py', '--db', str(self.database),
|
|
'--backup-dir', str(self.backups), *args], capture_output=True, text=True,
|
|
encoding='utf-8', timeout=15)
|
|
|
|
def test_backup_and_check_create_verified_snapshot(self):
|
|
result = self.run_cli('backup')
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
backup = Path(result.stdout.strip().split(': ', 1)[-1])
|
|
self.assertTrue(backup.is_file())
|
|
checked = self.run_cli('check', '--input', str(backup))
|
|
self.assertEqual(checked.returncode, 0, checked.stderr)
|
|
self.assertIn('ok', checked.stdout.lower())
|
|
|
|
def test_restore_keeps_current_safety_backup_and_replaces_database(self):
|
|
source = self.backups/'source.db'
|
|
with closing(sqlite3.connect(source)) as connection:
|
|
connection.execute('CREATE TABLE records(value TEXT NOT NULL)')
|
|
connection.execute("INSERT INTO records VALUES('restored')")
|
|
connection.commit()
|
|
result = self.run_cli('restore', '--input', str(source), '--service-stopped')
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
with closing(sqlite3.connect(self.database)) as connection:
|
|
self.assertEqual(connection.execute('SELECT value FROM records').fetchone()[0], 'restored')
|
|
self.assertTrue(list(self.backups.glob('pre-restore-*.db')))
|
|
|
|
def test_restore_rejects_a_backup_outside_configured_directory(self):
|
|
outside = self.root/'outside.db'
|
|
outside.write_bytes(self.database.read_bytes())
|
|
result = self.run_cli('restore', '--input', str(outside), '--service-stopped')
|
|
self.assertNotEqual(result.returncode, 0)
|
|
with closing(sqlite3.connect(self.database)) as connection:
|
|
self.assertEqual(connection.execute('SELECT value FROM records').fetchone()[0], 'original')
|
|
|
|
def test_restore_requires_explicit_stopped_service_declaration(self):
|
|
result = self.run_cli('restore', '--input', str(self.backups/'source.db'))
|
|
self.assertNotEqual(result.returncode, 0)
|
|
|
|
def test_corrupt_backup_cannot_overwrite_current_data(self):
|
|
corrupt = self.backups/'corrupt.db'
|
|
corrupt.write_bytes(b'not sqlite')
|
|
result = self.run_cli('restore', '--input', str(corrupt), '--service-stopped')
|
|
self.assertNotEqual(result.returncode, 0)
|
|
with closing(sqlite3.connect(self.database)) as connection:
|
|
self.assertEqual(connection.execute('SELECT value FROM records').fetchone()[0], 'original')
|
|
|
|
def test_wal_records_are_present_and_snapshot_does_not_overwrite_existing_file(self):
|
|
destination = self.backups/'wal.db'
|
|
with closing(sqlite3.connect(self.database)) as connection:
|
|
connection.execute('PRAGMA journal_mode=WAL')
|
|
connection.execute("INSERT INTO records VALUES('committed-wal')")
|
|
connection.commit()
|
|
database_backup.snapshot_database(self.database, destination)
|
|
with closing(sqlite3.connect(destination)) as copied:
|
|
self.assertEqual(copied.execute('SELECT COUNT(*) FROM records').fetchone()[0], 2)
|
|
with self.assertRaises(FileExistsError):
|
|
database_backup.snapshot_database(self.database, destination)
|
|
self.assertEqual(list(self.backups.glob('*.tmp*')), [])
|
|
|
|
def test_restore_of_clean_wal_database_does_not_leave_or_reuse_sidecars(self):
|
|
with closing(sqlite3.connect(self.database)) as connection:
|
|
connection.execute('PRAGMA journal_mode=WAL')
|
|
source = self.backups/'source.db'
|
|
database_backup.snapshot_database(self.database, source)
|
|
# A read-only WAL source can leave empty coordination sidecars.
|
|
with closing(sqlite3.connect(self.database)) as connection:
|
|
connection.execute('PRAGMA wal_checkpoint(TRUNCATE)')
|
|
database_backup.restore_database(self.database, source, self.backups)
|
|
for suffix in ('-wal','-shm'):
|
|
self.assertFalse(Path(str(self.database)+suffix).exists())
|
|
database_backup.restore_database(self.database, source, self.backups)
|
|
|
|
def test_daily_worker_works_without_requests_retries_failures_and_deduplicates(self):
|
|
from flask import Flask
|
|
app = Flask(__name__)
|
|
app.config.update(DATABASE=self.database, BACKUP_DIR=self.backups)
|
|
worker = database_backup.DailyBackupWorker(app)
|
|
tomorrow = datetime.now(database_backup.CHINA_TIME) + timedelta(days=1)
|
|
with patch('logistics_chat.database_backup.daily_backup', side_effect=OSError('fixture disk full')):
|
|
with self.assertRaises(OSError):
|
|
worker.tick(tomorrow)
|
|
self.assertTrue(worker.tick(tomorrow))
|
|
self.assertFalse(worker.tick(tomorrow))
|
|
self.assertEqual(len(list(self.backups.glob('history_*.db'))), 1)
|
|
|
|
def test_missing_database_and_invalid_snapshot_are_never_created_or_published(self):
|
|
missing = self.root/'missing.db'
|
|
with self.assertRaises(sqlite3.Error):
|
|
database_backup.snapshot_database(missing, self.backups/'bad.db')
|
|
self.assertFalse(missing.exists())
|
|
self.assertFalse((self.backups/'bad.db').exists())
|
|
self.assertEqual(list(self.backups.glob('*.tmp')), [])
|
|
|
|
def test_daily_retention_leaves_upgrade_and_manual_backups_untouched(self):
|
|
for name in ('history_20000101.db','pre-migration-old.db','manual-old.db','history_invalid.db'):
|
|
(self.backups/name).write_bytes(b'fixture')
|
|
database_backup.daily_backup(self.database, self.backups)
|
|
self.assertFalse((self.backups/'history_20000101.db').exists())
|
|
for name in ('pre-migration-old.db','manual-old.db','history_invalid.db'):
|
|
self.assertTrue((self.backups/name).exists())
|
|
|
|
def test_daily_backup_keeps_exactly_seven_calendar_dates(self):
|
|
today = datetime.now(database_backup.CHINA_TIME)
|
|
for offset in range(1, 9):
|
|
(self.backups/('history_' + (today-timedelta(days=offset)).strftime('%Y%m%d') + '.db')).write_bytes(b'old')
|
|
database_backup.daily_backup(self.database, self.backups, today)
|
|
self.assertEqual(len(list(self.backups.glob('history_*.db'))), 7)
|
|
|
|
def test_migration_retention_removes_only_oldest_recognized_snapshots(self):
|
|
names = []
|
|
for index in range(7):
|
|
name = f'pre-migration-20261102-12000{index}-000000-{index:08x}.db'
|
|
names.append(name)
|
|
(self.backups/name).write_bytes(b'fixture')
|
|
(self.backups/'pre-migration-old.db').write_bytes(b'keep')
|
|
(self.backups/'manual-old.db').write_bytes(b'keep')
|
|
(self.backups/'pre-migration-20261102-120000-000000-badname.db').write_bytes(b'keep')
|
|
(self.backups/'pre-migration-20261102-120000-000000-ffffffff.db').mkdir()
|
|
|
|
database_backup.prune_migration_snapshots(self.backups, keep=5)
|
|
|
|
self.assertEqual([name for name in names if (self.backups/name).exists()], names[2:])
|
|
self.assertTrue((self.backups/'pre-migration-old.db').exists())
|
|
self.assertTrue((self.backups/'manual-old.db').exists())
|
|
self.assertTrue((self.backups/'pre-migration-20261102-120000-000000-badname.db').exists())
|
|
self.assertTrue((self.backups/'pre-migration-20261102-120000-000000-ffffffff.db').is_dir())
|