"""Offline, verifiable SQLite -> EMPTY PostgreSQL database migration (V8.18). Data only. Shared files (/data/media, builds, archives, legacy) must be moved separately, under the same maintenance window, before site activation. This never clears target rows. PostgreSQL target schema is bootstrapped by SiteForge before copying; application must be fully stopped in both installations. """ from __future__ import annotations import hashlib import json import re import sqlite3 import tempfile from pathlib import Path from .pg_disaster_recovery import RecoveryError IDENT=re.compile(r'^[a-z_][a-z0-9_]*$',re.I) LEDGER='schema_migrations' def quote(name): if not IDENT.fullmatch(name): raise RecoveryError('Unsupported table or column identifier') return '"'+name+'"' def plan_tables(connection: sqlite3.Connection) -> list[str]: tables=[r[0] for r in connection.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")] if 'sites' not in tables:raise RecoveryError('SQLite source has no sites table') graph={t:set() for t in tables} for table in tables: for dep in connection.execute(f'PRAGMA foreign_key_list({quote(table)})'): parent=dep[2] if parent != table: if parent not in graph:raise RecoveryError('Missing FK table '+parent) graph[table].add(parent) result=[] while graph: ready=sorted(name for name,deps in graph.items() if not deps) if not ready:raise RecoveryError('Circular foreign keys require separate migration') for name in ready: result.append(name);graph.pop(name) for refs in graph.values():refs.difference_update(ready) return result def columns(conn: sqlite3.Connection, table: str) -> list[str]: return [row[1] for row in conn.execute('PRAGMA table_info('+quote(table)+')')] def _hash_rows(rows) -> str: """Order independent deterministic content hash, for migration verification.""" canonical=[] for row in rows: canonical.append(json.dumps(list(row),sort_keys=True,ensure_ascii=False, separators=(',',':'),default=str)) digest=hashlib.sha256() for line in sorted(canonical):digest.update((line+'\n').encode('utf-8')) return digest.hexdigest() def bootstrap_target(): # Same ordered migrations as FastAPI lifespan and PostgreSQL bootstrap smoke. from . import (db,enterprise,factory_api,migration_audit,reusables,release_api, cms_pipeline,v8_workflows,v8_templates,wp_migration_v81,wp_migration_v82, wp_migration_v83,connectors_v84,connectors_v85,studio_v86,site_design_v87,multilingual) if not db.POSTGRES:raise RecoveryError('SITEFORGE_DATABASE_URL must point at PostgreSQL') db.init() for module in (enterprise,factory_api,migration_audit,reusables,release_api, cms_pipeline,v8_workflows,v8_templates,wp_migration_v81,wp_migration_v82, wp_migration_v83,connectors_v84,connectors_v85,studio_v86,site_design_v87,multilingual): module.setup() def check_destination_empty(pg, source: sqlite3.Connection, tables: list[str]) -> None: for table in tables: name=quote(table) # Check source column coverage. Unknown source columns mean unsafe truncation. target_cols={r['column_name'] for r in pg.execute( 'SELECT column_name FROM information_schema.columns WHERE table_schema=current_schema() AND table_name=?', (table,)).fetchall()} src_cols=set(columns(source,table)) if not src_cols or not src_cols.issubset(target_cols): raise RecoveryError('PostgreSQL schema mismatch at table '+table) if table == LEDGER: # Destination is bootstrapped and has its own required migration ledger. old=source.execute('SELECT version,checksum FROM schema_migrations').fetchall() for version,checksum in old: row=pg.execute('SELECT checksum FROM schema_migrations WHERE version=?',(version,)).fetchone() if not row or row[0]!=checksum: raise RecoveryError('Migration ledger mismatch: '+version) continue count=pg.execute(f'SELECT COUNT(*) FROM {name}').fetchone()[0] if count: raise RecoveryError('Destination is not empty: '+table) def transfer_rows(pg, source, tables, batch=250) -> dict: if not (1<=batch<=500):raise RecoveryError('Batch size must be between 1 and 500') result={} for table in tables: if table==LEDGER:continue cols=columns(source,table) select=f'SELECT {", ".join(map(quote,cols))} FROM {quote(table)}' original=source.execute(select) n=0 # Keep below PostgreSQL's 65,535 bind-parameter limit. limit=min(batch,max(1,65000//len(cols))) while True: rows=original.fetchmany(limit) if not rows:break placeholders='('+','.join('?' for _ in cols)+')' insert=f'INSERT INTO {quote(table)} ({", ".join(map(quote,cols))}) VALUES '+','.join([placeholders]*len(rows)) pg.execute(insert,[value for row in rows for value in row]) n+=len(rows) source_hash=_hash_rows(source.execute(select)) destination_rows=pg.execute(select).fetchall() destination_hash=_hash_rows([[row[col] for col in cols] for row in destination_rows]) if n!=len(destination_rows) or source_hash!=destination_hash: raise RecoveryError('Data validation failed at table '+table) result[table]={'rows':n,'sha256':source_hash} return result def migrate(source_path: str | Path, *, confirmed_offline=False, batch=250, connect=None, bootstrap=True) -> dict: """Caller supplies PG connection in tests; production uses app.db.connect.""" if not confirmed_offline:raise RecoveryError('Stop all services and use --confirmed-offline') src=Path(source_path).resolve() if not src.is_file():raise RecoveryError('SQLite source file not found') from . import db if connect is None and not db.POSTGRES: raise RecoveryError('PostgreSQL URL is required') # Capture an immutable consistent snapshot; does not create or change source DB. with tempfile.TemporaryDirectory(prefix='siteforge-sqlite-pg-') as temp: snapshot=Path(temp)/'snapshot.sqlite3' with sqlite3.connect(src.as_uri()+'?mode=ro',uri=True) as old, sqlite3.connect(snapshot) as new: old.backup(new) with sqlite3.connect(snapshot.as_uri()+'?mode=ro',uri=True) as sqlite: if sqlite.execute('PRAGMA integrity_check').fetchone()[0]!='ok': raise RecoveryError('SQLite integrity check failed') ordered=plan_tables(sqlite) if bootstrap:bootstrap_target() connect_fn=connect or db.connect # All data inserts and verification occur inside ONE PG transaction. with connect_fn() as target: check_destination_empty(target,sqlite,ordered) counts=transfer_rows(target,sqlite,ordered,batch) return {'source':'sqlite', 'destination':'postgresql', 'tables':len(counts),'records':sum(x['rows'] for x in counts.values()), 'verified':True,'details':counts, 'note':'Database only. Transfer shared files before traffic cutover.'}