chore: baseline V8.10.0-dev.1 with v8_workflows type-mapping fix and trade-pro spec
- SiteForge Enterprise V8.10.0-dev.1 upstream source (210 tests) - Fix: v8_workflows._mapped_input now reads both 类型/分类 columns so WXR imports keep post/product types instead of collapsing to page - Docs: trade-pro theme port spec (R3) and implementation plan Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -0,0 +1,282 @@
|
||||
"""V8.3: checkpointed public WordPress category/tag/media catalog and staged references.
|
||||
|
||||
Catalogs are *metadata only*, not a full fidelity migration of WordPress custom
|
||||
fields, image files, taxonomy archive templates or WooCommerce variations.
|
||||
All remote traffic uses the pinned safe_http client. Nothing here publishes a site.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import json
|
||||
import re
|
||||
from urllib.parse import urlencode, urlsplit
|
||||
from fastapi import APIRouter, Body, HTTPException, Request
|
||||
from . import db, enterprise, safe_http, wp_migration_v81 as wp
|
||||
|
||||
router = APIRouter(prefix='/api/migrations/sites', tags=['V8.3 WordPress catalog'])
|
||||
RESOURCE = {'categories': 'category', 'tags': 'tag', 'media': 'media'}
|
||||
PAGE_SIZE = 50
|
||||
MAX_ROWS = 5000
|
||||
MAX_STORED_META = 16000
|
||||
|
||||
|
||||
def setup():
|
||||
with db.connect() as c:
|
||||
c.executescript('''
|
||||
CREATE TABLE IF NOT EXISTS wp_catalog_cursors(
|
||||
session_id TEXT NOT NULL REFERENCES wp_migration_sessions(id) ON DELETE CASCADE,
|
||||
resource TEXT NOT NULL, next_page INTEGER NOT NULL DEFAULT 1,
|
||||
fetched INTEGER NOT NULL DEFAULT 0, done INTEGER NOT NULL DEFAULT 0,
|
||||
last_error TEXT NOT NULL DEFAULT '', updated_at TEXT NOT NULL,
|
||||
PRIMARY KEY(session_id,resource));
|
||||
CREATE TABLE IF NOT EXISTS wp_catalog_items(
|
||||
session_id TEXT NOT NULL REFERENCES wp_migration_sessions(id) ON DELETE CASCADE,
|
||||
resource TEXT NOT NULL, wp_id INTEGER NOT NULL,
|
||||
name TEXT NOT NULL DEFAULT '', slug TEXT NOT NULL DEFAULT '',
|
||||
parent_id INTEGER NOT NULL DEFAULT 0, source_url TEXT NOT NULL DEFAULT '',
|
||||
meta_json TEXT NOT NULL DEFAULT '{}',
|
||||
PRIMARY KEY(session_id,resource,wp_id));
|
||||
CREATE INDEX IF NOT EXISTS wp_catalog_session_resource
|
||||
ON wp_catalog_items(session_id,resource);
|
||||
CREATE TABLE IF NOT EXISTS wp_catalog_refs(
|
||||
session_id TEXT NOT NULL REFERENCES wp_migration_sessions(id) ON DELETE CASCADE,
|
||||
source_key TEXT NOT NULL, resource TEXT NOT NULL,
|
||||
target_id INTEGER NOT NULL,
|
||||
PRIMARY KEY(session_id,source_key,resource,target_id));
|
||||
CREATE INDEX IF NOT EXISTS wp_catalog_refs_by_target
|
||||
ON wp_catalog_refs(session_id,resource,target_id);
|
||||
''')
|
||||
|
||||
|
||||
def _safe_id_list(values):
|
||||
if not isinstance(values, list):return []
|
||||
return list(dict.fromkeys(v for v in values[:100] if type(v) is int and 0 < v <= 2_000_000_000))
|
||||
|
||||
|
||||
def _source_meta(source, phase):
|
||||
"""Bounded public-source snapshot kept separate from SiteForge business copy."""
|
||||
meta = {'platform':'wordpress','wp_id':source['id'],'source_type':phase,
|
||||
'category_ids':_safe_id_list(source.get('categories')),
|
||||
'tag_ids':_safe_id_list(source.get('tags')),
|
||||
'featured_media_id':source.get('featured_media') if type(source.get('featured_media')) is int else 0}
|
||||
acf=source.get('acf')
|
||||
if isinstance(acf,dict):
|
||||
packed=json.dumps(acf,ensure_ascii=False)
|
||||
meta['acf_keys']=[str(k)[:100] for k in list(acf)[:80]]
|
||||
if len(packed.encode('utf-8'))<=MAX_STORED_META:
|
||||
meta['acf_snapshot']=acf
|
||||
else:meta['acf_truncated']=True
|
||||
meta['requires_builder_review']=bool(source.get('elementor_data') or source.get('_elementor_data'))
|
||||
return meta
|
||||
|
||||
|
||||
def _record(resource, obj):
|
||||
if not isinstance(obj,dict) or type(obj.get('id')) is not int or obj['id']<1:
|
||||
raise ValueError('WordPress 分类/标签/媒体记录缺少有效 ID')
|
||||
raw_url=obj.get('source_url','') if resource=='media' else obj.get('link','')
|
||||
meta={}
|
||||
if resource=='media':
|
||||
meta={'mime_type':str(obj.get('mime_type') or '')[:120],
|
||||
'alt_text':str(obj.get('alt_text') or '')[:500],
|
||||
'caption':str((obj.get('caption') or {}).get('rendered',''))[:1000] if isinstance(obj.get('caption'),dict) else '',
|
||||
'media_details':{'width':(obj.get('media_details') or {}).get('width'),
|
||||
'height':(obj.get('media_details') or {}).get('height')}}
|
||||
title=obj.get('title')
|
||||
name=obj.get('name') or (title.get('rendered','') if isinstance(title,dict) else '')
|
||||
return (int(obj['id']),str(name or '')[:350],
|
||||
str(obj.get('slug') or '')[:200],int(obj.get('parent') or 0) if resource=='categories' else 0,
|
||||
str(raw_url or '')[:1000],json.dumps(meta,ensure_ascii=False))
|
||||
|
||||
|
||||
def _fetch(origin,resource,page):
|
||||
url=origin+'wp-json/wp/v2/'+resource+'?'+urlencode({'per_page':PAGE_SIZE,'page':page})
|
||||
resp=safe_http.fetch(url,max_bytes=4_000_000,timeout=18,redirects=0,headers={'Accept':'application/json'})
|
||||
if resp.status_code==400 and page>1:
|
||||
# End of pagination only for REST invalid-page-number, not arbitrary 400s.
|
||||
try: code=resp.json().get('code')
|
||||
except (ValueError,AttributeError):code=None
|
||||
if code=='rest_post_invalid_page_number':return []
|
||||
resp.raise_for_status()
|
||||
rows=resp.json()
|
||||
if not isinstance(rows,list) or len(rows)>PAGE_SIZE:raise ValueError('REST 分类目录返回数据不符合单页限制')
|
||||
return rows
|
||||
|
||||
|
||||
def sync_page(site_id,session_id,resource):
|
||||
s=wp._one(site_id,session_id)
|
||||
if resource not in RESOURCE:raise ValueError('不支持的 WordPress 资源目录')
|
||||
if s['state'] in ('running','cancelled','completed','completed_with_warnings'):
|
||||
raise HTTPException(409,'当前会话状态不允许采集分类目录')
|
||||
cursor=db.one('SELECT * FROM wp_catalog_cursors WHERE session_id=? AND resource=?',(session_id,resource))
|
||||
if cursor and cursor['done']:return catalog_status(site_id,session_id)
|
||||
page=cursor['next_page'] if cursor else 1
|
||||
rows=_fetch(s['origin'],resource,page)
|
||||
parsed=[_record(resource,obj) for obj in rows]
|
||||
if (cursor['fetched'] if cursor else 0)+len(rows)>MAX_ROWS:
|
||||
raise ValueError('单个分类目录最多采集 5000 条；请分批处理')
|
||||
# Atomic cursor progression: a failure never skips an uncommitted page.
|
||||
with db.connect() as c:
|
||||
c.execute('BEGIN IMMEDIATE')
|
||||
actual=c.execute('SELECT next_page,done FROM wp_catalog_cursors WHERE session_id=? AND resource=?',
|
||||
(session_id,resource)).fetchone()
|
||||
if (actual and (actual['next_page']!=page or actual['done'])) or (not actual and page!=1):
|
||||
# Another operator already committed the same page. Never count it twice.
|
||||
return catalog_status(site_id,session_id)
|
||||
for wp_id,name,slug,parent,source,meta in parsed:
|
||||
c.execute('''INSERT INTO wp_catalog_items(session_id,resource,wp_id,name,slug,parent_id,source_url,meta_json)
|
||||
VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(session_id,resource,wp_id) DO UPDATE SET
|
||||
name=excluded.name,slug=excluded.slug,parent_id=excluded.parent_id,
|
||||
source_url=excluded.source_url,meta_json=excluded.meta_json''',
|
||||
(session_id,resource,wp_id,name,slug,parent,source,meta))
|
||||
c.execute('''INSERT INTO wp_catalog_cursors(session_id,resource,next_page,fetched,done,updated_at)
|
||||
VALUES(?,?,?,?,?,?) ON CONFLICT(session_id,resource) DO UPDATE SET
|
||||
next_page=excluded.next_page,fetched=excluded.fetched,done=excluded.done,updated_at=excluded.updated_at''',
|
||||
(session_id,resource,page+1,(cursor['fetched'] if cursor else 0)+len(rows),int(len(rows)<PAGE_SIZE),db.now()))
|
||||
refresh_staged_relations(site_id,session_id)
|
||||
return catalog_status(site_id,session_id)
|
||||
|
||||
|
||||
def catalog_status(site_id,session_id):
|
||||
wp._one(site_id,session_id)
|
||||
states={r['resource']:{'next_page':r['next_page'],'fetched':r['fetched'],'done':bool(r['done'])}
|
||||
for r in db.rows('SELECT * FROM wp_catalog_cursors WHERE session_id=?',(session_id,))}
|
||||
for resource in RESOURCE:
|
||||
states.setdefault(resource,{'next_page':1,'fetched':0,'done':False,'not_started':True})
|
||||
return {'session_id':session_id,'resources':states,
|
||||
'term_counts':{r:db.one('SELECT COUNT(*) n FROM wp_catalog_items WHERE session_id=? AND resource=?',
|
||||
(session_id,r))['n'] for r in RESOURCE}}
|
||||
|
||||
|
||||
def staged_relations(site_id,session_id,source_rows,phase,batch_id):
|
||||
"""Add WP references to staged items only; never overwrite committed edits."""
|
||||
if not batch_id:return
|
||||
sources={f'wp-rest:{phase}:{s["id"]}':s for s in source_rows if isinstance(s,dict) and type(s.get('id')) is int}
|
||||
with db.connect() as c:
|
||||
c.execute('BEGIN IMMEDIATE')
|
||||
items=c.execute('SELECT * FROM v8_import_items WHERE batch_id=? AND state=?',(batch_id,'staged')).fetchall()
|
||||
for item in items:
|
||||
src=sources.get(item['source_key'])
|
||||
if not src:continue
|
||||
content=json.loads(item['content_json']);meta=content.get('meta') or {}
|
||||
meta['wp_source']=_source_meta(src,phase)
|
||||
content['meta']=meta
|
||||
for resource,ids in (('categories',meta['wp_source']['category_ids']),('tags',meta['wp_source']['tag_ids']),
|
||||
('media',[meta['wp_source']['featured_media_id']] if meta['wp_source']['featured_media_id'] else [])):
|
||||
for target_id in ids:
|
||||
c.execute('INSERT OR IGNORE INTO wp_catalog_refs(session_id,source_key,resource,target_id) VALUES(?,?,?,?)',
|
||||
(session_id,item['source_key'],resource,target_id))
|
||||
c.execute('UPDATE v8_import_items SET content_json=? WHERE id=?',
|
||||
(json.dumps(content,ensure_ascii=False),item['id']))
|
||||
refresh_staged_relations(site_id,session_id)
|
||||
|
||||
|
||||
def refresh_staged_relations(site_id,session_id):
|
||||
wp._one(site_id,session_id)
|
||||
terms={kind:{r['wp_id']:r for r in db.rows('SELECT * FROM wp_catalog_items WHERE session_id=? AND resource=?',
|
||||
(session_id,kind))} for kind in ('categories','tags','media')}
|
||||
batches=db.rows('''SELECT DISTINCT p.batch_id FROM wp_migration_pages p
|
||||
JOIN v8_import_batches b ON b.id=p.batch_id WHERE p.session_id=? AND b.site_id=? AND b.state='staged' ''',
|
||||
(session_id,site_id))
|
||||
changed=0
|
||||
for batch in batches:
|
||||
if not batch['batch_id']:continue
|
||||
with db.connect() as c:
|
||||
c.execute('BEGIN IMMEDIATE')
|
||||
for row in c.execute('SELECT * FROM v8_import_items WHERE batch_id=? AND state=?',(batch['batch_id'],'staged')).fetchall():
|
||||
content=json.loads(row['content_json']);meta=content.get('meta') or {}
|
||||
wp_source=meta.get('wp_source') or {}
|
||||
if not wp_source:continue
|
||||
fields=meta.setdefault('fields',{})
|
||||
for kind,key in (('categories','category'),('tags','tags')):
|
||||
names=[terms[kind][i]['name'] for i in wp_source.get('category_ids' if kind=='categories' else 'tag_ids',[])
|
||||
if i in terms[kind] and terms[kind][i]['name']]
|
||||
if names:
|
||||
fields[key]=names[0] if key=='category' else ', '.join(names[:30])
|
||||
wp_source[kind+'_resolved']=names
|
||||
mid=wp_source.get('featured_media_id')
|
||||
if mid and mid in terms['media']:
|
||||
wp_source['featured_media_original']=terms['media'][mid]['source_url']
|
||||
text=json.dumps(content,ensure_ascii=False)
|
||||
if text!=row['content_json']:
|
||||
c.execute('UPDATE v8_import_items SET content_json=? WHERE id=?',(text,row['id']))
|
||||
changed+=1
|
||||
return {'updated':changed}
|
||||
|
||||
|
||||
def register_archive_baseline(site_id,session_id):
|
||||
"""Add taxonomy archive URLs to the source audit, without fabricating pages."""
|
||||
from .core import route_path
|
||||
session=wp._one(site_id,session_id)
|
||||
base=urlsplit(session['origin'])
|
||||
added=0
|
||||
with db.connect() as c:
|
||||
c.execute('BEGIN IMMEDIATE')
|
||||
items=c.execute("SELECT resource,source_url FROM wp_catalog_items WHERE session_id=? AND resource IN ('categories','tags')",(session_id,)).fetchall()
|
||||
for term in items:
|
||||
url=term['source_url']
|
||||
if not url:continue
|
||||
p=urlsplit(url)
|
||||
if (p.scheme!=base.scheme or p.hostname!=base.hostname or p.port!=base.port or p.username or p.password
|
||||
or p.query or p.fragment or not p.path.startswith(base.path)):
|
||||
continue
|
||||
try:route=route_path(p.path)
|
||||
except ValueError:continue
|
||||
if c.execute('SELECT route FROM wp_migration_urls WHERE session_id=? AND route=?',(session_id,route)).fetchone():
|
||||
continue
|
||||
c.execute('INSERT INTO wp_migration_urls(session_id,route,original_url,status) VALUES(?,?,?,200)',
|
||||
(session_id,route,url))
|
||||
added+=1
|
||||
return {'taxonomy_archives_added':added}
|
||||
|
||||
|
||||
def findings(site_id,session_id):
|
||||
"""All discovered references must be either matched or explicitly reviewed."""
|
||||
wp._one(site_id,session_id)
|
||||
seen=[]
|
||||
terms={kind:{r['wp_id'] for r in db.rows('SELECT wp_id FROM wp_catalog_items WHERE session_id=? AND resource=?',
|
||||
(session_id,kind))} for kind in RESOURCE}
|
||||
batches=db.rows('SELECT batch_id FROM wp_migration_pages WHERE session_id=? AND batch_id IS NOT NULL',(session_id,))
|
||||
for b in batches:
|
||||
for r in db.rows('SELECT route,content_json FROM v8_import_items WHERE batch_id=?',(b['batch_id'],)):
|
||||
obj=json.loads(r['content_json']);source=(obj.get('meta') or {}).get('wp_source') or {}
|
||||
for kind,key in (('categories','category_ids'),('tags','tag_ids')):
|
||||
for n in source.get(key,[]):
|
||||
if n not in terms[kind]:seen.append(f'缺少 WordPress {kind} #{n}，内容 {r["route"]}')
|
||||
mid=source.get('featured_media_id')
|
||||
if mid and mid not in terms['media']:
|
||||
seen.append(f'特色图片媒体记录 #{mid} 未核对，内容 {r["route"]}')
|
||||
if source.get('acf_keys'):
|
||||
seen.append(f'ACF 自定义字段需要人工核对 {r["route"]}: '+', '.join(source['acf_keys'][:8]))
|
||||
if source.get('acf_truncated'):
|
||||
seen.append(f'ACF 内容过大未保留完整快照 {r["route"]}')
|
||||
if source.get('requires_builder_review'):
|
||||
seen.append(f'页面构建器布局需要人工复核 {r["route"]}')
|
||||
# Broken parent references are explicit review issues.
|
||||
base=urlsplit(wp._one(site_id,session_id)['origin'])
|
||||
for item in db.rows('SELECT resource,wp_id,name,parent_id,source_url FROM wp_catalog_items WHERE session_id=?',(session_id,)):
|
||||
if item['resource']=='categories' and item['parent_id'] and item['parent_id'] not in terms['categories']:
|
||||
seen.append(f'分类 #{item["wp_id"]} 的上级分类 #{item["parent_id"]} 未采集')
|
||||
if item['resource'] in ('categories','tags') and not item['name']:
|
||||
seen.append(f'WordPress {item["resource"]} #{item["wp_id"]} 缺少名称')
|
||||
if item['resource'] in ('categories','tags') and item['source_url']:
|
||||
parsed=urlsplit(item['source_url'])
|
||||
if parsed.scheme!=base.scheme or parsed.hostname!=base.hostname or parsed.port!=base.port:
|
||||
seen.append(f'WordPress 分类/标签归档 URL 与源站不同源：{item["source_url"][:150]}')
|
||||
return list(dict.fromkeys(seen))[:500]
|
||||
|
||||
|
||||
@router.get('/{site_id}/wordpress/sessions/{session_id}/catalog')
|
||||
def api_catalog_status(request:Request,site_id:str,session_id:str):
|
||||
enterprise.require(request,site_id,'viewer')
|
||||
return catalog_status(site_id,session_id)
|
||||
|
||||
|
||||
@router.post('/{site_id}/wordpress/sessions/{session_id}/catalog/{resource}/next')
|
||||
def api_catalog_next(request:Request,site_id:str,session_id:str,resource:str):
|
||||
enterprise.require(request,site_id,'editor')
|
||||
try:return sync_page(site_id,session_id,resource)
|
||||
except (ValueError,safe_http.OutboundError) as exc:raise HTTPException(400,str(exc)) from exc
|
||||
|
||||
|
||||
@router.post('/{site_id}/wordpress/sessions/{session_id}/catalog/refresh')
|
||||
def api_catalog_refresh(request:Request,site_id:str,session_id:str,payload:dict=Body(default_factory=dict)):
|
||||
enterprise.require(request,site_id,'editor')
|
||||
return refresh_staged_relations(site_id,session_id)
|
||||
Reference in New Issue
Block a user