upgrade: adopt V8.21.3 (V8.13→8.21.3: P5 SFTP/S3, P6 multilingual, editor overhaul, enterprise hardening)

Overlay ~150 upstream items onto our V8.12 base, preserving trade-pro theme, registry tests, and design docs. Re-applied our patch set (discovery/env_for/blocks.css/type-mapping/theme_color/frontend entries, test_theme_pro_v3 mods) — all anchors survived upstream evolution. New platform enhancement (fixes upgrade-introduced regression): multilingual.site_locale now honors settings_json.locale, plus PUT /api/i18n/sites/{id}/locale — standalone English sites get <html lang='en'> under the V8.15 locale-rewrite pipeline. compose: adopt V8.21.3 (image 8.21.3-dev.1, /api/ready probes, MySQL/S3 env) with project name pinned to siteforge-v8-10 (data volumes). Tests: 356 passed / 11 failed — all 11 confirmed pre-existing in pristine V8.21.3 on this host (10 upstream Windows env + our build test now green after locale fix). Patch registry updated in plan doc. Co-Authored-By: Claude Code <noreply@anthropic.com>
2026-10-23 12:31:03 +08:00
parent 257c89e159
commit 601451f22b
218 changed files with 10441 additions and 628 deletions
@@ -0,0 +1,154 @@
+"""Offline, verifiable SQLite -> EMPTY PostgreSQL database migration (V8.18).
+
+Data only. Shared files (/data/media, builds, archives, legacy) must be moved
+separately, under the same maintenance window, before site activation.
+
+This never clears target rows. PostgreSQL target schema is bootstrapped by SiteForge
+before copying; application must be fully stopped in both installations.
+"""
+from __future__ import annotations
+import hashlib
+import json
+import re
+import sqlite3
+import tempfile
+from pathlib import Path
+
+from .pg_disaster_recovery import RecoveryError
+
+IDENT=re.compile(r'^[a-z_][a-z0-9_]*$',re.I)
+LEDGER='schema_migrations'
+
+
+def quote(name):
+    if not IDENT.fullmatch(name):
+        raise RecoveryError('Unsupported table or column identifier')
+    return '"'+name+'"'
+
+
+def plan_tables(connection: sqlite3.Connection) -> list[str]:
+    tables=[r[0] for r in connection.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")]
+    if 'sites' not in tables:raise RecoveryError('SQLite source has no sites table')
+    graph={t:set() for t in tables}
+    for table in tables:
+        for dep in connection.execute(f'PRAGMA foreign_key_list({quote(table)})'):
+            parent=dep[2]
+            if parent != table:
+                if parent not in graph:raise RecoveryError('Missing FK table '+parent)
+                graph[table].add(parent)
+    result=[]
+    while graph:
+        ready=sorted(name for name,deps in graph.items() if not deps)
+        if not ready:raise RecoveryError('Circular foreign keys require separate migration')
+        for name in ready:
+            result.append(name);graph.pop(name)
+        for refs in graph.values():refs.difference_update(ready)
+    return result
+
+
+def columns(conn: sqlite3.Connection, table: str) -> list[str]:
+    return [row[1] for row in conn.execute('PRAGMA table_info('+quote(table)+')')]
+
+
+def _hash_rows(rows) -> str:
+    """Order independent deterministic content hash, for migration verification."""
+    canonical=[]
+    for row in rows:
+        canonical.append(json.dumps(list(row),sort_keys=True,ensure_ascii=False,
+                                    separators=(',',':'),default=str))
+    digest=hashlib.sha256()
+    for line in sorted(canonical):digest.update((line+'\n').encode('utf-8'))
+    return digest.hexdigest()
+
+
+def bootstrap_target():
+    # Same ordered migrations as FastAPI lifespan and PostgreSQL bootstrap smoke.
+    from . import (db,enterprise,factory_api,migration_audit,reusables,release_api,
+        cms_pipeline,v8_workflows,v8_templates,wp_migration_v81,wp_migration_v82,
+        wp_migration_v83,connectors_v84,connectors_v85,studio_v86,site_design_v87,multilingual)
+    if not db.POSTGRES:raise RecoveryError('SITEFORGE_DATABASE_URL must point at PostgreSQL')
+    db.init()
+    for module in (enterprise,factory_api,migration_audit,reusables,release_api,
+        cms_pipeline,v8_workflows,v8_templates,wp_migration_v81,wp_migration_v82,
+        wp_migration_v83,connectors_v84,connectors_v85,studio_v86,site_design_v87,multilingual):
+        module.setup()
+
+
+def check_destination_empty(pg, source: sqlite3.Connection, tables: list[str]) -> None:
+    for table in tables:
+        name=quote(table)
+        # Check source column coverage. Unknown source columns mean unsafe truncation.
+        target_cols={r['column_name'] for r in pg.execute(
+            'SELECT column_name FROM information_schema.columns WHERE table_schema=current_schema() AND table_name=?',
+            (table,)).fetchall()}
+        src_cols=set(columns(source,table))
+        if not src_cols or not src_cols.issubset(target_cols):
+            raise RecoveryError('PostgreSQL schema mismatch at table '+table)
+        if table == LEDGER:
+            # Destination is bootstrapped and has its own required migration ledger.
+            old=source.execute('SELECT version,checksum FROM schema_migrations').fetchall()
+            for version,checksum in old:
+                row=pg.execute('SELECT checksum FROM schema_migrations WHERE version=?',(version,)).fetchone()
+                if not row or row[0]!=checksum:
+                    raise RecoveryError('Migration ledger mismatch: '+version)
+            continue
+        count=pg.execute(f'SELECT COUNT(*) FROM {name}').fetchone()[0]
+        if count:
+            raise RecoveryError('Destination is not empty: '+table)
+
+
+def transfer_rows(pg, source, tables, batch=250) -> dict:
+    if not (1<=batch<=500):raise RecoveryError('Batch size must be between 1 and 500')
+    result={}
+    for table in tables:
+        if table==LEDGER:continue
+        cols=columns(source,table)
+        select=f'SELECT {", ".join(map(quote,cols))} FROM {quote(table)}'
+        original=source.execute(select)
+        n=0
+        # Keep below PostgreSQL's 65,535 bind-parameter limit.
+        limit=min(batch,max(1,65000//len(cols)))
+        while True:
+            rows=original.fetchmany(limit)
+            if not rows:break
+            placeholders='('+','.join('?' for _ in cols)+')'
+            insert=f'INSERT INTO {quote(table)} ({", ".join(map(quote,cols))}) VALUES '+','.join([placeholders]*len(rows))
+            pg.execute(insert,[value for row in rows for value in row])
+            n+=len(rows)
+        source_hash=_hash_rows(source.execute(select))
+        destination_rows=pg.execute(select).fetchall()
+        destination_hash=_hash_rows([[row[col] for col in cols] for row in destination_rows])
+        if n!=len(destination_rows) or source_hash!=destination_hash:
+            raise RecoveryError('Data validation failed at table '+table)
+        result[table]={'rows':n,'sha256':source_hash}
+    return result
+
+
+def migrate(source_path: str | Path, *, confirmed_offline=False, batch=250,
+            connect=None, bootstrap=True) -> dict:
+    """Caller supplies PG connection in tests; production uses app.db.connect."""
+    if not confirmed_offline:raise RecoveryError('Stop all services and use --confirmed-offline')
+    src=Path(source_path).resolve()
+    if not src.is_file():raise RecoveryError('SQLite source file not found')
+    from . import db
+    if connect is None and not db.POSTGRES:
+        raise RecoveryError('PostgreSQL URL is required')
+    # Capture an immutable consistent snapshot; does not create or change source DB.
+    with tempfile.TemporaryDirectory(prefix='siteforge-sqlite-pg-') as temp:
+        snapshot=Path(temp)/'snapshot.sqlite3'
+        with sqlite3.connect(src.as_uri()+'?mode=ro',uri=True) as old, sqlite3.connect(snapshot) as new:
+            old.backup(new)
+        with sqlite3.connect(snapshot.as_uri()+'?mode=ro',uri=True) as sqlite:
+            if sqlite.execute('PRAGMA integrity_check').fetchone()[0]!='ok':
+                raise RecoveryError('SQLite integrity check failed')
+            ordered=plan_tables(sqlite)
+            if bootstrap:bootstrap_target()
+            connect_fn=connect or db.connect
+            # All data inserts and verification occur inside ONE PG transaction.
+            with connect_fn() as target:
+                check_destination_empty(target,sqlite,ordered)
+                counts=transfer_rows(target,sqlite,ordered,batch)
+    return {'source':'sqlite', 'destination':'postgresql',
+            'tables':len(counts),'records':sum(x['rows'] for x in counts.values()),
+            'verified':True,'details':counts,
+            'note':'Database only. Transfer shared files before traffic cutover.'}