283 lines
16 KiB
Python
283 lines
16 KiB
Python
|
"""V8.3: checkpointed public WordPress category/tag/media catalog and staged references.
|
||
|
|
|
||
|
|
Catalogs are *metadata only*, not a full fidelity migration of WordPress custom
|
||
|
|
fields, image files, taxonomy archive templates or WooCommerce variations.
|
||
|
|
All remote traffic uses the pinned safe_http client. Nothing here publishes a site.
|
||
|
|
"""
|
||
|
|
from __future__ import annotations
|
||
|
|
import json
|
||
|
|
import re
|
||
|
|
from urllib.parse import urlencode, urlsplit
|
||
|
|
from fastapi import APIRouter, Body, HTTPException, Request
|
||
|
|
from . import db, enterprise, safe_http, wp_migration_v81 as wp
|
||
|
|
|
||
|
|
router = APIRouter(prefix='/api/migrations/sites', tags=['V8.3 WordPress catalog'])
|
||
|
|
RESOURCE = {'categories': 'category', 'tags': 'tag', 'media': 'media'}
|
||
|
|
PAGE_SIZE = 50
|
||
|
|
MAX_ROWS = 5000
|
||
|
|
MAX_STORED_META = 16000
|
||
|
|
|
||
|
|
|
||
|
|
def setup():
|
||
|
|
with db.connect() as c:
|
||
|
|
c.executescript('''
|
||
|
|
CREATE TABLE IF NOT EXISTS wp_catalog_cursors(
|
||
|
|
session_id TEXT NOT NULL REFERENCES wp_migration_sessions(id) ON DELETE CASCADE,
|
||
|
|
resource TEXT NOT NULL, next_page INTEGER NOT NULL DEFAULT 1,
|
||
|
|
fetched INTEGER NOT NULL DEFAULT 0, done INTEGER NOT NULL DEFAULT 0,
|
||
|
|
last_error TEXT NOT NULL DEFAULT '', updated_at TEXT NOT NULL,
|
||
|
|
PRIMARY KEY(session_id,resource));
|
||
|
|
CREATE TABLE IF NOT EXISTS wp_catalog_items(
|
||
|
|
session_id TEXT NOT NULL REFERENCES wp_migration_sessions(id) ON DELETE CASCADE,
|
||
|
|
resource TEXT NOT NULL, wp_id INTEGER NOT NULL,
|
||
|
|
name TEXT NOT NULL DEFAULT '', slug TEXT NOT NULL DEFAULT '',
|
||
|
|
parent_id INTEGER NOT NULL DEFAULT 0, source_url TEXT NOT NULL DEFAULT '',
|
||
|
|
meta_json TEXT NOT NULL DEFAULT '{}',
|
||
|
|
PRIMARY KEY(session_id,resource,wp_id));
|
||
|
|
CREATE INDEX IF NOT EXISTS wp_catalog_session_resource
|
||
|
|
ON wp_catalog_items(session_id,resource);
|
||
|
|
CREATE TABLE IF NOT EXISTS wp_catalog_refs(
|
||
|
|
session_id TEXT NOT NULL REFERENCES wp_migration_sessions(id) ON DELETE CASCADE,
|
||
|
|
source_key TEXT NOT NULL, resource TEXT NOT NULL,
|
||
|
|
target_id INTEGER NOT NULL,
|
||
|
|
PRIMARY KEY(session_id,source_key,resource,target_id));
|
||
|
|
CREATE INDEX IF NOT EXISTS wp_catalog_refs_by_target
|
||
|
|
ON wp_catalog_refs(session_id,resource,target_id);
|
||
|
|
''')
|
||
|
|
|
||
|
|
|
||
|
|
def _safe_id_list(values):
|
||
|
|
if not isinstance(values, list):return []
|
||
|
|
return list(dict.fromkeys(v for v in values[:100] if type(v) is int and 0 < v <= 2_000_000_000))
|
||
|
|
|
||
|
|
|
||
|
|
def _source_meta(source, phase):
|
||
|
|
"""Bounded public-source snapshot kept separate from SiteForge business copy."""
|
||
|
|
meta = {'platform':'wordpress','wp_id':source['id'],'source_type':phase,
|
||
|
|
'category_ids':_safe_id_list(source.get('categories')),
|
||
|
|
'tag_ids':_safe_id_list(source.get('tags')),
|
||
|
|
'featured_media_id':source.get('featured_media') if type(source.get('featured_media')) is int else 0}
|
||
|
|
acf=source.get('acf')
|
||
|
|
if isinstance(acf,dict):
|
||
|
|
packed=json.dumps(acf,ensure_ascii=False)
|
||
|
|
meta['acf_keys']=[str(k)[:100] for k in list(acf)[:80]]
|
||
|
|
if len(packed.encode('utf-8'))<=MAX_STORED_META:
|
||
|
|
meta['acf_snapshot']=acf
|
||
|
|
else:meta['acf_truncated']=True
|
||
|
|
meta['requires_builder_review']=bool(source.get('elementor_data') or source.get('_elementor_data'))
|
||
|
|
return meta
|
||
|
|
|
||
|
|
|
||
|
|
def _record(resource, obj):
|
||
|
|
if not isinstance(obj,dict) or type(obj.get('id')) is not int or obj['id']<1:
|
||
|
|
raise ValueError('WordPress 分类/标签/媒体记录缺少有效 ID')
|
||
|
|
raw_url=obj.get('source_url','') if resource=='media' else obj.get('link','')
|
||
|
|
meta={}
|
||
|
|
if resource=='media':
|
||
|
|
meta={'mime_type':str(obj.get('mime_type') or '')[:120],
|
||
|
|
'alt_text':str(obj.get('alt_text') or '')[:500],
|
||
|
|
'caption':str((obj.get('caption') or {}).get('rendered',''))[:1000] if isinstance(obj.get('caption'),dict) else '',
|
||
|
|
'media_details':{'width':(obj.get('media_details') or {}).get('width'),
|
||
|
|
'height':(obj.get('media_details') or {}).get('height')}}
|
||
|
|
title=obj.get('title')
|
||
|
|
name=obj.get('name') or (title.get('rendered','') if isinstance(title,dict) else '')
|
||
|
|
return (int(obj['id']),str(name or '')[:350],
|
||
|
|
str(obj.get('slug') or '')[:200],int(obj.get('parent') or 0) if resource=='categories' else 0,
|
||
|
|
str(raw_url or '')[:1000],json.dumps(meta,ensure_ascii=False))
|
||
|
|
|
||
|
|
|
||
|
|
def _fetch(origin,resource,page):
|
||
|
|
url=origin+'wp-json/wp/v2/'+resource+'?'+urlencode({'per_page':PAGE_SIZE,'page':page})
|
||
|
|
resp=safe_http.fetch(url,max_bytes=4_000_000,timeout=18,redirects=0,headers={'Accept':'application/json'})
|
||
|
|
if resp.status_code==400 and page>1:
|
||
|
|
# End of pagination only for REST invalid-page-number, not arbitrary 400s.
|
||
|
|
try: code=resp.json().get('code')
|
||
|
|
except (ValueError,AttributeError):code=None
|
||
|
|
if code=='rest_post_invalid_page_number':return []
|
||
|
|
resp.raise_for_status()
|
||
|
|
rows=resp.json()
|
||
|
|
if not isinstance(rows,list) or len(rows)>PAGE_SIZE:raise ValueError('REST 分类目录返回数据不符合单页限制')
|
||
|
|
return rows
|
||
|
|
|
||
|
|
|
||
|
|
def sync_page(site_id,session_id,resource):
|
||
|
|
s=wp._one(site_id,session_id)
|
||
|
|
if resource not in RESOURCE:raise ValueError('不支持的 WordPress 资源目录')
|
||
|
|
if s['state'] in ('running','cancelled','completed','completed_with_warnings'):
|
||
|
|
raise HTTPException(409,'当前会话状态不允许采集分类目录')
|
||
|
|
cursor=db.one('SELECT * FROM wp_catalog_cursors WHERE session_id=? AND resource=?',(session_id,resource))
|
||
|
|
if cursor and cursor['done']:return catalog_status(site_id,session_id)
|
||
|
|
page=cursor['next_page'] if cursor else 1
|
||
|
|
rows=_fetch(s['origin'],resource,page)
|
||
|
|
parsed=[_record(resource,obj) for obj in rows]
|
||
|
|
if (cursor['fetched'] if cursor else 0)+len(rows)>MAX_ROWS:
|
||
|
|
raise ValueError('单个分类目录最多采集 5000 条；请分批处理')
|
||
|
|
# Atomic cursor progression: a failure never skips an uncommitted page.
|
||
|
|
with db.connect() as c:
|
||
|
|
c.execute('BEGIN IMMEDIATE')
|
||
|
|
actual=c.execute('SELECT next_page,done FROM wp_catalog_cursors WHERE session_id=? AND resource=?',
|
||
|
|
(session_id,resource)).fetchone()
|
||
|
|
if (actual and (actual['next_page']!=page or actual['done'])) or (not actual and page!=1):
|
||
|
|
# Another operator already committed the same page. Never count it twice.
|
||
|
|
return catalog_status(site_id,session_id)
|
||
|
|
for wp_id,name,slug,parent,source,meta in parsed:
|
||
|
|
c.execute('''INSERT INTO wp_catalog_items(session_id,resource,wp_id,name,slug,parent_id,source_url,meta_json)
|
||
|
|
VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(session_id,resource,wp_id) DO UPDATE SET
|
||
|
|
name=excluded.name,slug=excluded.slug,parent_id=excluded.parent_id,
|
||
|
|
source_url=excluded.source_url,meta_json=excluded.meta_json''',
|
||
|
|
(session_id,resource,wp_id,name,slug,parent,source,meta))
|
||
|
|
c.execute('''INSERT INTO wp_catalog_cursors(session_id,resource,next_page,fetched,done,updated_at)
|
||
|
|
VALUES(?,?,?,?,?,?) ON CONFLICT(session_id,resource) DO UPDATE SET
|
||
|
|
next_page=excluded.next_page,fetched=excluded.fetched,done=excluded.done,updated_at=excluded.updated_at''',
|
||
|
|
(session_id,resource,page+1,(cursor['fetched'] if cursor else 0)+len(rows),int(len(rows)<PAGE_SIZE),db.now()))
|
||
|
|
refresh_staged_relations(site_id,session_id)
|
||
|
|
return catalog_status(site_id,session_id)
|
||
|
|
|
||
|
|
|
||
|
|
def catalog_status(site_id,session_id):
|
||
|
|
wp._one(site_id,session_id)
|
||
|
|
states={r['resource']:{'next_page':r['next_page'],'fetched':r['fetched'],'done':bool(r['done'])}
|
||
|
|
for r in db.rows('SELECT * FROM wp_catalog_cursors WHERE session_id=?',(session_id,))}
|
||
|
|
for resource in RESOURCE:
|
||
|
|
states.setdefault(resource,{'next_page':1,'fetched':0,'done':False,'not_started':True})
|
||
|
|
return {'session_id':session_id,'resources':states,
|
||
|
|
'term_counts':{r:db.one('SELECT COUNT(*) n FROM wp_catalog_items WHERE session_id=? AND resource=?',
|
||
|
|
(session_id,r))['n'] for r in RESOURCE}}
|
||
|
|
|
||
|
|
|
||
|
|
def staged_relations(site_id,session_id,source_rows,phase,batch_id):
|
||
|
|
"""Add WP references to staged items only; never overwrite committed edits."""
|
||
|
|
if not batch_id:return
|
||
|
|
sources={f'wp-rest:{phase}:{s["id"]}':s for s in source_rows if isinstance(s,dict) and type(s.get('id')) is int}
|
||
|
|
with db.connect() as c:
|
||
|
|
c.execute('BEGIN IMMEDIATE')
|
||
|
|
items=c.execute('SELECT * FROM v8_import_items WHERE batch_id=? AND state=?',(batch_id,'staged')).fetchall()
|
||
|
|
for item in items:
|
||
|
|
src=sources.get(item['source_key'])
|
||
|
|
if not src:continue
|
||
|
|
content=json.loads(item['content_json']);meta=content.get('meta') or {}
|
||
|
|
meta['wp_source']=_source_meta(src,phase)
|
||
|
|
content['meta']=meta
|
||
|
|
for resource,ids in (('categories',meta['wp_source']['category_ids']),('tags',meta['wp_source']['tag_ids']),
|
||
|
|
('media',[meta['wp_source']['featured_media_id']] if meta['wp_source']['featured_media_id'] else [])):
|
||
|
|
for target_id in ids:
|
||
|
|
c.execute('INSERT OR IGNORE INTO wp_catalog_refs(session_id,source_key,resource,target_id) VALUES(?,?,?,?)',
|
||
|
|
(session_id,item['source_key'],resource,target_id))
|
||
|
|
c.execute('UPDATE v8_import_items SET content_json=? WHERE id=?',
|
||
|
|
(json.dumps(content,ensure_ascii=False),item['id']))
|
||
|
|
refresh_staged_relations(site_id,session_id)
|
||
|
|
|
||
|
|
|
||
|
|
def refresh_staged_relations(site_id,session_id):
|
||
|
|
wp._one(site_id,session_id)
|
||
|
|
terms={kind:{r['wp_id']:r for r in db.rows('SELECT * FROM wp_catalog_items WHERE session_id=? AND resource=?',
|
||
|
|
(session_id,kind))} for kind in ('categories','tags','media')}
|
||
|
|
batches=db.rows('''SELECT DISTINCT p.batch_id FROM wp_migration_pages p
|
||
|
|
JOIN v8_import_batches b ON b.id=p.batch_id WHERE p.session_id=? AND b.site_id=? AND b.state='staged' ''',
|
||
|
|
(session_id,site_id))
|
||
|
|
changed=0
|
||
|
|
for batch in batches:
|
||
|
|
if not batch['batch_id']:continue
|
||
|
|
with db.connect() as c:
|
||
|
|
c.execute('BEGIN IMMEDIATE')
|
||
|
|
for row in c.execute('SELECT * FROM v8_import_items WHERE batch_id=? AND state=?',(batch['batch_id'],'staged')).fetchall():
|
||
|
|
content=json.loads(row['content_json']);meta=content.get('meta') or {}
|
||
|
|
wp_source=meta.get('wp_source') or {}
|
||
|
|
if not wp_source:continue
|
||
|
|
fields=meta.setdefault('fields',{})
|
||
|
|
for kind,key in (('categories','category'),('tags','tags')):
|
||
|
|
names=[terms[kind][i]['name'] for i in wp_source.get('category_ids' if kind=='categories' else 'tag_ids',[])
|
||
|
|
if i in terms[kind] and terms[kind][i]['name']]
|
||
|
|
if names:
|
||
|
|
fields[key]=names[0] if key=='category' else ', '.join(names[:30])
|
||
|
|
wp_source[kind+'_resolved']=names
|
||
|
|
mid=wp_source.get('featured_media_id')
|
||
|
|
if mid and mid in terms['media']:
|
||
|
|
wp_source['featured_media_original']=terms['media'][mid]['source_url']
|
||
|
|
text=json.dumps(content,ensure_ascii=False)
|
||
|
|
if text!=row['content_json']:
|
||
|
|
c.execute('UPDATE v8_import_items SET content_json=? WHERE id=?',(text,row['id']))
|
||
|
|
changed+=1
|
||
|
|
return {'updated':changed}
|
||
|
|
|
||
|
|
|
||
|
|
def register_archive_baseline(site_id,session_id):
|
||
|
|
"""Add taxonomy archive URLs to the source audit, without fabricating pages."""
|
||
|
|
from .core import route_path
|
||
|
|
session=wp._one(site_id,session_id)
|
||
|
|
base=urlsplit(session['origin'])
|
||
|
|
added=0
|
||
|
|
with db.connect() as c:
|
||
|
|
c.execute('BEGIN IMMEDIATE')
|
||
|
|
items=c.execute("SELECT resource,source_url FROM wp_catalog_items WHERE session_id=? AND resource IN ('categories','tags')",(session_id,)).fetchall()
|
||
|
|
for term in items:
|
||
|
|
url=term['source_url']
|
||
|
|
if not url:continue
|
||
|
|
p=urlsplit(url)
|
||
|
|
if (p.scheme!=base.scheme or p.hostname!=base.hostname or p.port!=base.port or p.username or p.password
|
||
|
|
or p.query or p.fragment or not p.path.startswith(base.path)):
|
||
|
|
continue
|
||
|
|
try:route=route_path(p.path)
|
||
|
|
except ValueError:continue
|
||
|
|
if c.execute('SELECT route FROM wp_migration_urls WHERE session_id=? AND route=?',(session_id,route)).fetchone():
|
||
|
|
continue
|
||
|
|
c.execute('INSERT INTO wp_migration_urls(session_id,route,original_url,status) VALUES(?,?,?,200)',
|
||
|
|
(session_id,route,url))
|
||
|
|
added+=1
|
||
|
|
return {'taxonomy_archives_added':added}
|
||
|
|
|
||
|
|
|
||
|
|
def findings(site_id,session_id):
|
||
|
|
"""All discovered references must be either matched or explicitly reviewed."""
|
||
|
|
wp._one(site_id,session_id)
|
||
|
|
seen=[]
|
||
|
|
terms={kind:{r['wp_id'] for r in db.rows('SELECT wp_id FROM wp_catalog_items WHERE session_id=? AND resource=?',
|
||
|
|
(session_id,kind))} for kind in RESOURCE}
|
||
|
|
batches=db.rows('SELECT batch_id FROM wp_migration_pages WHERE session_id=? AND batch_id IS NOT NULL',(session_id,))
|
||
|
|
for b in batches:
|
||
|
|
for r in db.rows('SELECT route,content_json FROM v8_import_items WHERE batch_id=?',(b['batch_id'],)):
|
||
|
|
obj=json.loads(r['content_json']);source=(obj.get('meta') or {}).get('wp_source') or {}
|
||
|
|
for kind,key in (('categories','category_ids'),('tags','tag_ids')):
|
||
|
|
for n in source.get(key,[]):
|
||
|
|
if n not in terms[kind]:seen.append(f'缺少 WordPress {kind} #{n}，内容 {r["route"]}')
|
||
|
|
mid=source.get('featured_media_id')
|
||
|
|
if mid and mid not in terms['media']:
|
||
|
|
seen.append(f'特色图片媒体记录 #{mid} 未核对，内容 {r["route"]}')
|
||
|
|
if source.get('acf_keys'):
|
||
|
|
seen.append(f'ACF 自定义字段需要人工核对 {r["route"]}: '+', '.join(source['acf_keys'][:8]))
|
||
|
|
if source.get('acf_truncated'):
|
||
|
|
seen.append(f'ACF 内容过大未保留完整快照 {r["route"]}')
|
||
|
|
if source.get('requires_builder_review'):
|
||
|
|
seen.append(f'页面构建器布局需要人工复核 {r["route"]}')
|
||
|
|
# Broken parent references are explicit review issues.
|
||
|
|
base=urlsplit(wp._one(site_id,session_id)['origin'])
|
||
|
|
for item in db.rows('SELECT resource,wp_id,name,parent_id,source_url FROM wp_catalog_items WHERE session_id=?',(session_id,)):
|
||
|
|
if item['resource']=='categories' and item['parent_id'] and item['parent_id'] not in terms['categories']:
|
||
|
|
seen.append(f'分类 #{item["wp_id"]} 的上级分类 #{item["parent_id"]} 未采集')
|
||
|
|
if item['resource'] in ('categories','tags') and not item['name']:
|
||
|
|
seen.append(f'WordPress {item["resource"]} #{item["wp_id"]} 缺少名称')
|
||
|
|
if item['resource'] in ('categories','tags') and item['source_url']:
|
||
|
|
parsed=urlsplit(item['source_url'])
|
||
|
|
if parsed.scheme!=base.scheme or parsed.hostname!=base.hostname or parsed.port!=base.port:
|
||
|
|
seen.append(f'WordPress 分类/标签归档 URL 与源站不同源：{item["source_url"][:150]}')
|
||
|
|
return list(dict.fromkeys(seen))[:500]
|
||
|
|
|
||
|
|
|
||
|
|
@router.get('/{site_id}/wordpress/sessions/{session_id}/catalog')
|
||
|
|
def api_catalog_status(request:Request,site_id:str,session_id:str):
|
||
|
|
enterprise.require(request,site_id,'viewer')
|
||
|
|
return catalog_status(site_id,session_id)
|
||
|
|
|
||
|
|
|
||
|
|
@router.post('/{site_id}/wordpress/sessions/{session_id}/catalog/{resource}/next')
|
||
|
|
def api_catalog_next(request:Request,site_id:str,session_id:str,resource:str):
|
||
|
|
enterprise.require(request,site_id,'editor')
|
||
|
|
try:return sync_page(site_id,session_id,resource)
|
||
|
|
except (ValueError,safe_http.OutboundError) as exc:raise HTTPException(400,str(exc)) from exc
|
||
|
|
|
||
|
|
|
||
|
|
@router.post('/{site_id}/wordpress/sessions/{session_id}/catalog/refresh')
|
||
|
|
def api_catalog_refresh(request:Request,site_id:str,session_id:str,payload:dict=Body(default_factory=dict)):
|
||
|
|
enterprise.require(request,site_id,'editor')
|
||
|
|
return refresh_staged_relations(site_id,session_id)
|