chore: baseline V8.10.0-dev.1 with v8_workflows type-mapping fix and trade-pro spec
- SiteForge Enterprise V8.10.0-dev.1 upstream source (210 tests) - Fix: v8_workflows._mapped_input now reads both 类型/分类 columns so WXR imports keep post/product types instead of collapsing to page - Docs: trade-pro theme port spec (R3) and implementation plan Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -0,0 +1,280 @@
|
||||
"""V8 portable content import pipeline: deterministic preview, batch audit and guarded rollback.
|
||||
|
||||
This is a single-instance pipeline suitable for controlled pilots. It never runs
|
||||
untrusted scripts, and keeps original import and business data separate.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
import csv
|
||||
from urllib.parse import urlsplit
|
||||
from fastapi import APIRouter, Body, HTTPException, Request, File, UploadFile, Form
|
||||
from . import db, enterprise, assembly
|
||||
from .core import save_entry, route_path
|
||||
|
||||
router = APIRouter(prefix='/api/factory/sites', tags=['V8 content workflow'])
|
||||
MAX_ITEMS = 1000
|
||||
MAX_UPLOAD = 12_000_000
|
||||
|
||||
def setup():
|
||||
with db.connect() as c:
|
||||
c.executescript('''
|
||||
CREATE TABLE IF NOT EXISTS v8_import_batches(
|
||||
id TEXT PRIMARY KEY, site_id TEXT NOT NULL REFERENCES sites(id) ON DELETE CASCADE,
|
||||
origin TEXT NOT NULL, content_type TEXT NOT NULL, state TEXT NOT NULL,
|
||||
mapping_json TEXT NOT NULL, warnings_json TEXT NOT NULL DEFAULT '[]',
|
||||
input_count INTEGER NOT NULL, created_at TEXT NOT NULL, committed_at TEXT,
|
||||
rolled_back_at TEXT);
|
||||
CREATE TABLE IF NOT EXISTS v8_import_items(
|
||||
id TEXT PRIMARY KEY, batch_id TEXT NOT NULL REFERENCES v8_import_batches(id) ON DELETE CASCADE,
|
||||
source_key TEXT NOT NULL, row_number INTEGER NOT NULL,
|
||||
route TEXT NOT NULL, content_json TEXT NOT NULL, state TEXT NOT NULL DEFAULT 'staged',
|
||||
entry_id TEXT, before_json TEXT, after_fingerprint TEXT, error TEXT,
|
||||
UNIQUE(batch_id, source_key));
|
||||
CREATE INDEX IF NOT EXISTS v8_batches_by_site ON v8_import_batches(site_id,created_at);
|
||||
CREATE TABLE IF NOT EXISTS v8_site_templates(
|
||||
id TEXT PRIMARY KEY, site_id TEXT NOT NULL REFERENCES sites(id) ON DELETE CASCADE,
|
||||
name TEXT NOT NULL, version TEXT NOT NULL, blueprint_json TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL);
|
||||
''')
|
||||
|
||||
def _row_hash(row):
|
||||
if row is None:return ''
|
||||
data={k:row[k] for k in ('id','site_id','type','title','route','slug','summary','body','cover',
|
||||
'seo_title','seo_description','canonical','robots','meta_json','status','source','created_at','updated_at')}
|
||||
return hashlib.sha256(json.dumps(data,ensure_ascii=False,sort_keys=True).encode()).hexdigest()
|
||||
|
||||
def _require_site(site_id):
|
||||
if not db.one('SELECT id FROM sites WHERE id=?',(site_id,)):
|
||||
raise HTTPException(404,'网站不存在')
|
||||
|
||||
def _mapped_input(rows, mapping, kind):
|
||||
if not isinstance(rows,list) or not rows or len(rows)>MAX_ITEMS:
|
||||
raise ValueError('每批需要 1-1000 条内容')
|
||||
if any(not isinstance(x,dict) for x in rows):raise ValueError('数据行必须是对象')
|
||||
if kind not in ('product','vehicle','page','post','case','brand','service','other','mixed'):
|
||||
raise ValueError('内容类型不受支持')
|
||||
mapping = mapping or assembly.suggest(list(rows[0].keys()))
|
||||
if not isinstance(mapping,dict):raise ValueError('映射规则不合法')
|
||||
mapped,errors=assembly.transform(rows,mapping,'page' if kind=='mixed' else kind)
|
||||
if kind=='mixed':
|
||||
kinds={'post':'post','page':'page','product':'product','brand':'brand','vehicle':'vehicle',
|
||||
'case':'case','service':'service'}
|
||||
for i,item in enumerate(mapped):
|
||||
src=str(rows[i].get('类型') or rows[i].get('分类') or 'page').lower()
|
||||
item['type']=kinds.get(src,'other')
|
||||
if errors:raise ValueError('数据预检失败：'+json.dumps(errors[:8],ensure_ascii=False))
|
||||
return mapped,mapping
|
||||
|
||||
def stage(site_id, rows, mapping=None, kind='product', origin='manual-import', source_keys=None):
|
||||
_require_site(site_id)
|
||||
mapped,mapping=_mapped_input(rows,mapping,kind)
|
||||
if len(json.dumps(mapped,ensure_ascii=False).encode('utf-8'))>MAX_UPLOAD:
|
||||
raise ValueError('映射后的批次超过12MB上限，请拆分导入')
|
||||
if source_keys is not None and len(source_keys)!=len(rows):raise ValueError('源记录主键数量不一致')
|
||||
existing_routes={x['route'] for x in db.rows('SELECT route FROM entries WHERE site_id=?',(site_id,))}
|
||||
conflicts=[x['route'] for x in mapped if x['route'] in existing_routes]
|
||||
batch_id=uuid.uuid4().hex
|
||||
now=db.now()
|
||||
keys=[]
|
||||
for i,item in enumerate(mapped):
|
||||
source_key=str(source_keys[i] if source_keys else item['route'])
|
||||
if not source_key or len(source_key)>250:raise ValueError('来源主键缺失或过长')
|
||||
keys.append(source_key)
|
||||
if len(set(keys))!=len(keys):raise ValueError('导入文件的来源主键重复')
|
||||
with db.connect() as c:
|
||||
c.execute('''INSERT INTO v8_import_batches(id,site_id,origin,content_type,state,mapping_json,
|
||||
input_count,created_at) VALUES(?,?,?,?,?,?,?,?)''',
|
||||
(batch_id,site_id,origin[:160],kind,'staged',json.dumps(mapping,ensure_ascii=False),len(mapped),now))
|
||||
for i,item in enumerate(mapped):
|
||||
c.execute('''INSERT INTO v8_import_items(id,batch_id,source_key,row_number,route,content_json)
|
||||
VALUES(?,?,?,?,?,?)''',(uuid.uuid4().hex,batch_id,keys[i],i+1,item['route'],
|
||||
json.dumps(item,ensure_ascii=False)))
|
||||
return {'batch_id':batch_id,'state':'staged','mapped':len(mapped),'conflicts':conflicts,
|
||||
'sample':mapped[:5],'mapping':mapping,'requires_conflict_choice':bool(conflicts)}
|
||||
|
||||
def list_batches(site_id):
|
||||
_require_site(site_id)
|
||||
return db.rows('SELECT id,origin,content_type,state,input_count,created_at,committed_at,rolled_back_at FROM v8_import_batches WHERE site_id=? ORDER BY created_at DESC LIMIT 100',(site_id,))
|
||||
|
||||
def batch_detail(site_id,batch_id):
|
||||
result=db.one('SELECT * FROM v8_import_batches WHERE site_id=? AND id=?',(site_id,batch_id))
|
||||
if not result:raise HTTPException(404,'导入批次不存在')
|
||||
result['items']=db.rows('SELECT id,source_key,row_number,route,state,entry_id,error FROM v8_import_items WHERE batch_id=? ORDER BY row_number LIMIT 1100',(batch_id,))
|
||||
return result
|
||||
|
||||
def commit(site_id,batch_id,on_conflict='skip'):
|
||||
if on_conflict not in ('skip','replace'):raise ValueError('冲突策略只能为 skip 或 replace')
|
||||
batch=db.one('SELECT * FROM v8_import_batches WHERE site_id=? AND id=?',(site_id,batch_id))
|
||||
if not batch:raise HTTPException(404,'导入批次不存在')
|
||||
if batch['state']=='committed':return {'batch_id':batch_id,'state':'committed','existing':True}
|
||||
if batch['state']!='staged':raise HTTPException(409,'批次已经开始处理，不能重复执行')
|
||||
locked=db.execute("UPDATE v8_import_batches SET state='processing' WHERE id=? AND state='staged'",(batch_id,))
|
||||
if not locked:raise HTTPException(409,'批次正在由其他请求处理')
|
||||
items=db.rows('SELECT * FROM v8_import_items WHERE batch_id=? ORDER BY row_number',(batch_id,))
|
||||
created=updated=skipped=0
|
||||
failed=[]
|
||||
for item in items:
|
||||
original=db.one('SELECT * FROM entries WHERE site_id=? AND route=?',(site_id,item['route']))
|
||||
if original and on_conflict=='skip':
|
||||
db.execute("UPDATE v8_import_items SET state='skipped', entry_id=? WHERE id=?",(original['id'],item['id']))
|
||||
skipped+=1
|
||||
continue
|
||||
try:
|
||||
payload=json.loads(item['content_json'])
|
||||
meta=payload.get('meta') or {}
|
||||
meta.setdefault('provenance',{})
|
||||
meta['provenance'].update({'import_batch':batch_id,'source_key':item['source_key'],'origin':batch['origin']})
|
||||
payload['meta']=meta
|
||||
before=json.dumps(original,ensure_ascii=False) if original else None
|
||||
eid=save_entry(site_id,payload,existing_id=original['id'] if original else None,source='v8-import')
|
||||
after=db.one('SELECT * FROM entries WHERE site_id=? AND id=?',(site_id,eid))
|
||||
db.execute('''UPDATE v8_import_items SET state='applied',entry_id=?,before_json=?,after_fingerprint=? WHERE id=?''',
|
||||
(eid,before,_row_hash(after),item['id']))
|
||||
if original:updated+=1
|
||||
else:created+=1
|
||||
except Exception as exc:
|
||||
msg=str(exc)[:250]
|
||||
db.execute("UPDATE v8_import_items SET state='error',error=? WHERE id=?",(msg,item['id']))
|
||||
failed.append({'row':item['row_number'],'message':msg})
|
||||
db.execute('UPDATE v8_import_batches SET state=?,committed_at=? WHERE id=?',
|
||||
('partial' if failed else 'committed',db.now(),batch_id))
|
||||
# Keep unsupported Elementor/WooCommerce details in the existing release gate.
|
||||
if batch['origin']=='wordpress-wxr':
|
||||
for finding in json.loads(batch.get('warnings_json') or '[]')[:200]:
|
||||
db.insert('''INSERT INTO cms_import_reviews(id,site_id,platform,row_number,issue,created_at)
|
||||
VALUES(?,?,?,?,?,?)''',(uuid.uuid4().hex,site_id,'wordpress-wxr',int(finding.get('row',0)),
|
||||
('v8:'+batch_id+': '+str(finding.get('issue','Requires review')))[:600],db.now()))
|
||||
return {'batch_id':batch_id,'state':'partial' if failed else 'committed',
|
||||
'created':created,'updated':updated,'skipped':skipped,'errors':failed}
|
||||
|
||||
def rollback(site_id,batch_id):
|
||||
# Hold one write transaction across the conflict check and all reversions.
|
||||
# A manual edit racing with rollback must not be accidentally overwritten.
|
||||
with db.connect() as c:
|
||||
c.execute('BEGIN IMMEDIATE')
|
||||
batch=c.execute('SELECT * FROM v8_import_batches WHERE site_id=? AND id=?',(site_id,batch_id)).fetchone()
|
||||
if not batch:raise HTTPException(404,'导入批次不存在')
|
||||
if batch['state']=='rolled_back':return {'batch_id':batch_id,'state':'rolled_back','existing':True}
|
||||
if batch['state'] not in ('committed','partial'):raise HTTPException(409,'批次还没有完成提交')
|
||||
applied=c.execute("SELECT * FROM v8_import_items WHERE batch_id=? AND state='applied' ORDER BY row_number DESC",(batch_id,)).fetchall()
|
||||
conflicts=[]
|
||||
for item in applied:
|
||||
current=c.execute('SELECT * FROM entries WHERE site_id=? AND id=?',(site_id,item['entry_id'])).fetchone()
|
||||
if _row_hash(dict(current) if current else None)!=item['after_fingerprint']:
|
||||
conflicts.append(item['route'])
|
||||
if conflicts:raise HTTPException(409,{'message':'内容在导入后被修改，拒绝覆盖人工编辑','routes':conflicts})
|
||||
cols=('type','title','route','slug','summary','body','cover','seo_title','seo_description',
|
||||
'canonical','robots','meta_json','status','source','created_at','updated_at')
|
||||
for item in applied:
|
||||
if not item['before_json']:
|
||||
c.execute('DELETE FROM entries WHERE site_id=? AND id=?',(site_id,item['entry_id']))
|
||||
else:
|
||||
before=json.loads(item['before_json'])
|
||||
assignments=','.join(k+'=?' for k in cols)
|
||||
c.execute(f'UPDATE entries SET {assignments} WHERE site_id=? AND id=?',
|
||||
(*[before[k] for k in cols],site_id,item['entry_id']))
|
||||
c.execute("UPDATE v8_import_items SET state='rolled_back' WHERE id=?",(item['id'],))
|
||||
c.execute("UPDATE v8_import_batches SET state='rolled_back',rolled_back_at=? WHERE id=?",(db.now(),batch_id))
|
||||
if batch['origin']=='wordpress-wxr':
|
||||
c.execute("UPDATE cms_import_reviews SET state='resolved',reviewer='system',resolution='Source import batch rolled back',resolved_at=? WHERE site_id=? AND platform='wordpress-wxr' AND state='pending' AND issue LIKE ?",
|
||||
(db.now(),site_id,'v8:'+batch_id+': %'))
|
||||
return {'batch_id':batch_id,'state':'rolled_back','restored':len(applied)}
|
||||
|
||||
def parse_wxr(content):
|
||||
"""Extract conservative WordPress records without writing into the site.
|
||||
Unsupported builder configuration stays in source metadata for review.
|
||||
"""
|
||||
from defusedxml import ElementTree as ET
|
||||
if len(content)>MAX_UPLOAD:raise ValueError('XML 文件超过限制')
|
||||
root=ET.fromstring(content)
|
||||
if root.tag!='rss':raise ValueError('这不是 WordPress WXR RSS 导出')
|
||||
rows=[];ids=[];warnings=[]
|
||||
for it in root.findall('./channel/item'):
|
||||
get=lambda name:next((str(node.text or '') for node in it if node.tag.endswith('}'+name) or node.tag==name),'')
|
||||
raw_typ=get('post_type') or 'post'
|
||||
status=get('status') or 'publish'
|
||||
if raw_typ in ('attachment','revision','nav_menu_item') or status not in ('publish','draft','private'):continue
|
||||
if len(rows)>=MAX_ITEMS:raise ValueError('单批 WXR 最多1000条，请拆分文件')
|
||||
wid=get('post_id')
|
||||
title=get('title').strip() or get('post_name').strip()
|
||||
if not title:continue
|
||||
url=get('link')
|
||||
slug=get('post_name') or re.sub(r'[^a-z0-9-]','-',title.lower())[:65] or uuid.uuid4().hex[:8]
|
||||
route=urlsplit(url).path if url.startswith(('http://','https://')) else ''
|
||||
if not route:route=f'/{slug}/' if raw_typ=='page' else f'/{raw_typ}/{slug}/'
|
||||
metas={}
|
||||
for pm in it:
|
||||
if not pm.tag.endswith('}postmeta'):continue
|
||||
k=v=''
|
||||
for n in pm:
|
||||
if n.tag.endswith('}meta_key'):k=n.text or ''
|
||||
if n.tag.endswith('}meta_value'):v=n.text or ''
|
||||
if k:metas[k]=v[:10000]
|
||||
if any(k in metas for k in ('_elementor_data','_product_attributes')):
|
||||
warnings.append({'row':len(rows)+1,'issue':'含复杂 Elementor/WooCommerce 组件，导入后需人工核对'})
|
||||
rows.append({'标题':title,'路径':route,'正文':get('encoded'),
|
||||
'SEO标题':metas.get('_yoast_wpseo_title') or metas.get('rank_math_title',''),
|
||||
'SEO描述':metas.get('_yoast_wpseo_metadesc') or metas.get('rank_math_description',''),
|
||||
'分类':raw_typ})
|
||||
ids.append(f'wordpress:{wid}' if wid else f'wordpress:path:{route}')
|
||||
return rows,ids,warnings
|
||||
|
||||
@router.post('/{site_id}/staging')
|
||||
def api_stage(request:Request,site_id:str,payload:dict=Body(...)):
|
||||
enterprise.require(request,site_id,'editor')
|
||||
try:return stage(site_id,payload.get('rows'),payload.get('mapping'),payload.get('content_type','product'),payload.get('origin','manual-import'))
|
||||
except ValueError as e:raise HTTPException(400,str(e))
|
||||
|
||||
@router.post('/{site_id}/staging/file')
|
||||
async def api_table_file(request:Request,site_id:str,file:UploadFile=File(...),
|
||||
content_type:str=Form('product')):
|
||||
enterprise.require(request,site_id,'editor')
|
||||
from .table_loader import parse_table
|
||||
blob=await file.read(MAX_UPLOAD+1)
|
||||
try:
|
||||
parsed=parse_table(file.filename or '',blob)
|
||||
result=stage(site_id,parsed['rows'],kind=content_type,origin='table:'+parsed['filename'][:120])
|
||||
result['columns']=parsed['columns']
|
||||
return result
|
||||
except (ValueError,UnicodeError,json.JSONDecodeError) as e:
|
||||
raise HTTPException(400,str(e))
|
||||
|
||||
@router.post('/{site_id}/staging/wxr')
|
||||
async def api_wxr(request:Request,site_id:str,file:UploadFile=File(...)):
|
||||
enterprise.require(request,site_id,'editor')
|
||||
if not (file.filename or '').lower().endswith('.xml'):raise HTTPException(400,'仅支持 .xml 导出')
|
||||
blob=await file.read(MAX_UPLOAD+1)
|
||||
try:
|
||||
rows,keys,warnings=parse_wxr(blob)
|
||||
if not rows:raise ValueError('没有可导入的 WordPress 页面或文章')
|
||||
result=stage(site_id,rows,{'title':'标题','route':'路径','body':'正文',
|
||||
'seo_title':'SEO标题','seo_description':'SEO描述'},'mixed','wordpress-wxr',keys)
|
||||
db.execute('UPDATE v8_import_batches SET warnings_json=? WHERE id=?',
|
||||
(json.dumps(warnings,ensure_ascii=False),result['batch_id']))
|
||||
return {**result,'warnings':warnings[:30]}
|
||||
except ValueError as e:raise HTTPException(400,str(e))
|
||||
|
||||
@router.get('/{site_id}/staging')
|
||||
def api_list(request:Request,site_id:str):
|
||||
enterprise.require(request,site_id,'viewer')
|
||||
return {'items':list_batches(site_id)}
|
||||
|
||||
@router.get('/{site_id}/staging/{batch_id}')
|
||||
def api_detail(request:Request,site_id:str,batch_id:str):
|
||||
enterprise.require(request,site_id,'viewer')
|
||||
return batch_detail(site_id,batch_id)
|
||||
|
||||
@router.post('/{site_id}/staging/{batch_id}/commit')
|
||||
def api_commit(request:Request,site_id:str,batch_id:str,payload:dict=Body(default_factory=dict)):
|
||||
enterprise.require(request,site_id,'editor')
|
||||
try:return commit(site_id,batch_id,payload.get('on_conflict','skip'))
|
||||
except ValueError as e:raise HTTPException(400,str(e))
|
||||
|
||||
@router.post('/{site_id}/staging/{batch_id}/rollback')
|
||||
def api_rollback(request:Request,site_id:str,batch_id:str):
|
||||
enterprise.require(request,site_id,'owner')
|
||||
return rollback(site_id,batch_id)
|
||||
Reference in New Issue
Block a user