cms-SiteForge/app/sqlite_pg_transfer.py

"""Offline, verifiable SQLite -> EMPTY PostgreSQL database migration (V8.18).

Data only. Shared files (/data/media, builds, archives, legacy) must be moved
separately, under the same maintenance window, before site activation.

This never clears target rows. PostgreSQL target schema is bootstrapped by SiteForge
before copying; application must be fully stopped in both installations.
"""
from __future__ import annotations
import hashlib
import json
import re
import sqlite3
import tempfile
from pathlib import Path

from .pg_disaster_recovery import RecoveryError

IDENT=re.compile(r'^[a-z_][a-z0-9_]*$',re.I)
LEDGER='schema_migrations'


def quote(name):
    if not IDENT.fullmatch(name):
        raise RecoveryError('Unsupported table or column identifier')
    return '"'+name+'"'


def plan_tables(connection: sqlite3.Connection) -> list[str]:
    tables=[r[0] for r in connection.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")]
    if 'sites' not in tables:raise RecoveryError('SQLite source has no sites table')
    graph={t:set() for t in tables}
    for table in tables:
        for dep in connection.execute(f'PRAGMA foreign_key_list({quote(table)})'):
            parent=dep[2]
            if parent != table:
                if parent not in graph:raise RecoveryError('Missing FK table '+parent)
                graph[table].add(parent)
    result=[]
    while graph:
        ready=sorted(name for name,deps in graph.items() if not deps)
        if not ready:raise RecoveryError('Circular foreign keys require separate migration')
        for name in ready:
            result.append(name);graph.pop(name)
        for refs in graph.values():refs.difference_update(ready)
    return result


def columns(conn: sqlite3.Connection, table: str) -> list[str]:
    return [row[1] for row in conn.execute('PRAGMA table_info('+quote(table)+')')]


def _hash_rows(rows) -> str:
    """Order independent deterministic content hash, for migration verification."""
    canonical=[]
    for row in rows:
        canonical.append(json.dumps(list(row),sort_keys=True,ensure_ascii=False,
                                    separators=(',',':'),default=str))
    digest=hashlib.sha256()
    for line in sorted(canonical):digest.update((line+'\n').encode('utf-8'))
    return digest.hexdigest()


def bootstrap_target():
    # Same ordered migrations as FastAPI lifespan and PostgreSQL bootstrap smoke.
    from . import (db,enterprise,factory_api,migration_audit,reusables,release_api,
        cms_pipeline,v8_workflows,v8_templates,wp_migration_v81,wp_migration_v82,
        wp_migration_v83,connectors_v84,connectors_v85,studio_v86,site_design_v87,multilingual)
    if not db.POSTGRES:raise RecoveryError('SITEFORGE_DATABASE_URL must point at PostgreSQL')
    db.init()
    for module in (enterprise,factory_api,migration_audit,reusables,release_api,
        cms_pipeline,v8_workflows,v8_templates,wp_migration_v81,wp_migration_v82,
        wp_migration_v83,connectors_v84,connectors_v85,studio_v86,site_design_v87,multilingual):
        module.setup()


def check_destination_empty(pg, source: sqlite3.Connection, tables: list[str]) -> None:
    for table in tables:
        name=quote(table)
        # Check source column coverage. Unknown source columns mean unsafe truncation.
        target_cols={r['column_name'] for r in pg.execute(
            'SELECT column_name FROM information_schema.columns WHERE table_schema=current_schema() AND table_name=?',
            (table,)).fetchall()}
        src_cols=set(columns(source,table))
        if not src_cols or not src_cols.issubset(target_cols):
            raise RecoveryError('PostgreSQL schema mismatch at table '+table)
        if table == LEDGER:
            # Destination is bootstrapped and has its own required migration ledger.
            old=source.execute('SELECT version,checksum FROM schema_migrations').fetchall()
            for version,checksum in old:
                row=pg.execute('SELECT checksum FROM schema_migrations WHERE version=?',(version,)).fetchone()
                if not row or row[0]!=checksum:
                    raise RecoveryError('Migration ledger mismatch: '+version)
            continue
        count=pg.execute(f'SELECT COUNT(*) FROM {name}').fetchone()[0]
        if count:
            raise RecoveryError('Destination is not empty: '+table)


def transfer_rows(pg, source, tables, batch=250) -> dict:
    if not (1<=batch<=500):raise RecoveryError('Batch size must be between 1 and 500')
    result={}
    for table in tables:
        if table==LEDGER:continue
        cols=columns(source,table)
        select=f'SELECT {", ".join(map(quote,cols))} FROM {quote(table)}'
        original=source.execute(select)
        n=0
        # Keep below PostgreSQL's 65,535 bind-parameter limit.
        limit=min(batch,max(1,65000//len(cols)))
        while True:
            rows=original.fetchmany(limit)
            if not rows:break
            placeholders='('+','.join('?' for _ in cols)+')'
            insert=f'INSERT INTO {quote(table)} ({", ".join(map(quote,cols))}) VALUES '+','.join([placeholders]*len(rows))
            pg.execute(insert,[value for row in rows for value in row])
            n+=len(rows)
        source_hash=_hash_rows(source.execute(select))
        destination_rows=pg.execute(select).fetchall()
        destination_hash=_hash_rows([[row[col] for col in cols] for row in destination_rows])
        if n!=len(destination_rows) or source_hash!=destination_hash:
            raise RecoveryError('Data validation failed at table '+table)
        result[table]={'rows':n,'sha256':source_hash}
    return result


def migrate(source_path: str | Path, *, confirmed_offline=False, batch=250,
            connect=None, bootstrap=True) -> dict:
    """Caller supplies PG connection in tests; production uses app.db.connect."""
    if not confirmed_offline:raise RecoveryError('Stop all services and use --confirmed-offline')
    src=Path(source_path).resolve()
    if not src.is_file():raise RecoveryError('SQLite source file not found')
    from . import db
    if connect is None and not db.POSTGRES:
        raise RecoveryError('PostgreSQL URL is required')
    # Capture an immutable consistent snapshot; does not create or change source DB.
    with tempfile.TemporaryDirectory(prefix='siteforge-sqlite-pg-') as temp:
        snapshot=Path(temp)/'snapshot.sqlite3'
        with sqlite3.connect(src.as_uri()+'?mode=ro',uri=True) as old, sqlite3.connect(snapshot) as new:
            old.backup(new)
        with sqlite3.connect(snapshot.as_uri()+'?mode=ro',uri=True) as sqlite:
            if sqlite.execute('PRAGMA integrity_check').fetchone()[0]!='ok':
                raise RecoveryError('SQLite integrity check failed')
            ordered=plan_tables(sqlite)
            if bootstrap:bootstrap_target()
            connect_fn=connect or db.connect
            # All data inserts and verification occur inside ONE PG transaction.
            with connect_fn() as target:
                check_destination_empty(target,sqlite,ordered)
                counts=transfer_rows(target,sqlite,ordered,batch)
    return {'source':'sqlite', 'destination':'postgresql',
            'tables':len(counts),'records':sum(x['rows'] for x in counts.values()),
            'verified':True,'details':counts,
            'note':'Database only. Transfer shared files before traffic cutover.'}