281 lines
16 KiB
Python
281 lines
16 KiB
Python
|
"""V8 portable content import pipeline: deterministic preview, batch audit and guarded rollback.
|
||
|
|
|
||
|
|
This is a single-instance pipeline suitable for controlled pilots. It never runs
|
||
|
|
untrusted scripts, and keeps original import and business data separate.
|
||
|
|
"""
|
||
|
|
from __future__ import annotations
|
||
|
|
import hashlib
|
||
|
|
import io
|
||
|
|
import json
|
||
|
|
import re
|
||
|
|
import uuid
|
||
|
|
import csv
|
||
|
|
from urllib.parse import urlsplit
|
||
|
|
from fastapi import APIRouter, Body, HTTPException, Request, File, UploadFile, Form
|
||
|
|
from . import db, enterprise, assembly
|
||
|
|
from .core import save_entry, route_path
|
||
|
|
|
||
|
|
router = APIRouter(prefix='/api/factory/sites', tags=['V8 content workflow'])
|
||
|
|
MAX_ITEMS = 1000
|
||
|
|
MAX_UPLOAD = 12_000_000
|
||
|
|
|
||
|
|
def setup():
|
||
|
|
with db.connect() as c:
|
||
|
|
c.executescript('''
|
||
|
|
CREATE TABLE IF NOT EXISTS v8_import_batches(
|
||
|
|
id TEXT PRIMARY KEY, site_id TEXT NOT NULL REFERENCES sites(id) ON DELETE CASCADE,
|
||
|
|
origin TEXT NOT NULL, content_type TEXT NOT NULL, state TEXT NOT NULL,
|
||
|
|
mapping_json TEXT NOT NULL, warnings_json TEXT NOT NULL DEFAULT '[]',
|
||
|
|
input_count INTEGER NOT NULL, created_at TEXT NOT NULL, committed_at TEXT,
|
||
|
|
rolled_back_at TEXT);
|
||
|
|
CREATE TABLE IF NOT EXISTS v8_import_items(
|
||
|
|
id TEXT PRIMARY KEY, batch_id TEXT NOT NULL REFERENCES v8_import_batches(id) ON DELETE CASCADE,
|
||
|
|
source_key TEXT NOT NULL, row_number INTEGER NOT NULL,
|
||
|
|
route TEXT NOT NULL, content_json TEXT NOT NULL, state TEXT NOT NULL DEFAULT 'staged',
|
||
|
|
entry_id TEXT, before_json TEXT, after_fingerprint TEXT, error TEXT,
|
||
|
|
UNIQUE(batch_id, source_key));
|
||
|
|
CREATE INDEX IF NOT EXISTS v8_batches_by_site ON v8_import_batches(site_id,created_at);
|
||
|
|
CREATE TABLE IF NOT EXISTS v8_site_templates(
|
||
|
|
id TEXT PRIMARY KEY, site_id TEXT NOT NULL REFERENCES sites(id) ON DELETE CASCADE,
|
||
|
|
name TEXT NOT NULL, version TEXT NOT NULL, blueprint_json TEXT NOT NULL,
|
||
|
|
created_at TEXT NOT NULL);
|
||
|
|
''')
|
||
|
|
|
||
|
|
def _row_hash(row):
|
||
|
|
if row is None:return ''
|
||
|
|
data={k:row[k] for k in ('id','site_id','type','title','route','slug','summary','body','cover',
|
||
|
|
'seo_title','seo_description','canonical','robots','meta_json','status','source','created_at','updated_at')}
|
||
|
|
return hashlib.sha256(json.dumps(data,ensure_ascii=False,sort_keys=True).encode()).hexdigest()
|
||
|
|
|
||
|
|
def _require_site(site_id):
|
||
|
|
if not db.one('SELECT id FROM sites WHERE id=?',(site_id,)):
|
||
|
|
raise HTTPException(404,'网站不存在')
|
||
|
|
|
||
|
|
def _mapped_input(rows, mapping, kind):
|
||
|
|
if not isinstance(rows,list) or not rows or len(rows)>MAX_ITEMS:
|
||
|
|
raise ValueError('每批需要 1-1000 条内容')
|
||
|
|
if any(not isinstance(x,dict) for x in rows):raise ValueError('数据行必须是对象')
|
||
|
|
if kind not in ('product','vehicle','page','post','case','brand','service','other','mixed'):
|
||
|
|
raise ValueError('内容类型不受支持')
|
||
|
|
mapping = mapping or assembly.suggest(list(rows[0].keys()))
|
||
|
|
if not isinstance(mapping,dict):raise ValueError('映射规则不合法')
|
||
|
|
mapped,errors=assembly.transform(rows,mapping,'page' if kind=='mixed' else kind)
|
||
|
|
if kind=='mixed':
|
||
|
|
kinds={'post':'post','page':'page','product':'product','brand':'brand','vehicle':'vehicle',
|
||
|
|
'case':'case','service':'service'}
|
||
|
|
for i,item in enumerate(mapped):
|
||
|
|
src=str(rows[i].get('类型') or rows[i].get('分类') or 'page').lower()
|
||
|
|
item['type']=kinds.get(src,'other')
|
||
|
|
if errors:raise ValueError('数据预检失败：'+json.dumps(errors[:8],ensure_ascii=False))
|
||
|
|
return mapped,mapping
|
||
|
|
|
||
|
|
def stage(site_id, rows, mapping=None, kind='product', origin='manual-import', source_keys=None):
|
||
|
|
_require_site(site_id)
|
||
|
|
mapped,mapping=_mapped_input(rows,mapping,kind)
|
||
|
|
if len(json.dumps(mapped,ensure_ascii=False).encode('utf-8'))>MAX_UPLOAD:
|
||
|
|
raise ValueError('映射后的批次超过12MB上限，请拆分导入')
|
||
|
|
if source_keys is not None and len(source_keys)!=len(rows):raise ValueError('源记录主键数量不一致')
|
||
|
|
existing_routes={x['route'] for x in db.rows('SELECT route FROM entries WHERE site_id=?',(site_id,))}
|
||
|
|
conflicts=[x['route'] for x in mapped if x['route'] in existing_routes]
|
||
|
|
batch_id=uuid.uuid4().hex
|
||
|
|
now=db.now()
|
||
|
|
keys=[]
|
||
|
|
for i,item in enumerate(mapped):
|
||
|
|
source_key=str(source_keys[i] if source_keys else item['route'])
|
||
|
|
if not source_key or len(source_key)>250:raise ValueError('来源主键缺失或过长')
|
||
|
|
keys.append(source_key)
|
||
|
|
if len(set(keys))!=len(keys):raise ValueError('导入文件的来源主键重复')
|
||
|
|
with db.connect() as c:
|
||
|
|
c.execute('''INSERT INTO v8_import_batches(id,site_id,origin,content_type,state,mapping_json,
|
||
|
|
input_count,created_at) VALUES(?,?,?,?,?,?,?,?)''',
|
||
|
|
(batch_id,site_id,origin[:160],kind,'staged',json.dumps(mapping,ensure_ascii=False),len(mapped),now))
|
||
|
|
for i,item in enumerate(mapped):
|
||
|
|
c.execute('''INSERT INTO v8_import_items(id,batch_id,source_key,row_number,route,content_json)
|
||
|
|
VALUES(?,?,?,?,?,?)''',(uuid.uuid4().hex,batch_id,keys[i],i+1,item['route'],
|
||
|
|
json.dumps(item,ensure_ascii=False)))
|
||
|
|
return {'batch_id':batch_id,'state':'staged','mapped':len(mapped),'conflicts':conflicts,
|
||
|
|
'sample':mapped[:5],'mapping':mapping,'requires_conflict_choice':bool(conflicts)}
|
||
|
|
|
||
|
|
def list_batches(site_id):
|
||
|
|
_require_site(site_id)
|
||
|
|
return db.rows('SELECT id,origin,content_type,state,input_count,created_at,committed_at,rolled_back_at FROM v8_import_batches WHERE site_id=? ORDER BY created_at DESC LIMIT 100',(site_id,))
|
||
|
|
|
||
|
|
def batch_detail(site_id,batch_id):
|
||
|
|
result=db.one('SELECT * FROM v8_import_batches WHERE site_id=? AND id=?',(site_id,batch_id))
|
||
|
|
if not result:raise HTTPException(404,'导入批次不存在')
|
||
|
|
result['items']=db.rows('SELECT id,source_key,row_number,route,state,entry_id,error FROM v8_import_items WHERE batch_id=? ORDER BY row_number LIMIT 1100',(batch_id,))
|
||
|
|
return result
|
||
|
|
|
||
|
|
def commit(site_id,batch_id,on_conflict='skip'):
|
||
|
|
if on_conflict not in ('skip','replace'):raise ValueError('冲突策略只能为 skip 或 replace')
|
||
|
|
batch=db.one('SELECT * FROM v8_import_batches WHERE site_id=? AND id=?',(site_id,batch_id))
|
||
|
|
if not batch:raise HTTPException(404,'导入批次不存在')
|
||
|
|
if batch['state']=='committed':return {'batch_id':batch_id,'state':'committed','existing':True}
|
||
|
|
if batch['state']!='staged':raise HTTPException(409,'批次已经开始处理，不能重复执行')
|
||
|
|
locked=db.execute("UPDATE v8_import_batches SET state='processing' WHERE id=? AND state='staged'",(batch_id,))
|
||
|
|
if not locked:raise HTTPException(409,'批次正在由其他请求处理')
|
||
|
|
items=db.rows('SELECT * FROM v8_import_items WHERE batch_id=? ORDER BY row_number',(batch_id,))
|
||
|
|
created=updated=skipped=0
|
||
|
|
failed=[]
|
||
|
|
for item in items:
|
||
|
|
original=db.one('SELECT * FROM entries WHERE site_id=? AND route=?',(site_id,item['route']))
|
||
|
|
if original and on_conflict=='skip':
|
||
|
|
db.execute("UPDATE v8_import_items SET state='skipped', entry_id=? WHERE id=?",(original['id'],item['id']))
|
||
|
|
skipped+=1
|
||
|
|
continue
|
||
|
|
try:
|
||
|
|
payload=json.loads(item['content_json'])
|
||
|
|
meta=payload.get('meta') or {}
|
||
|
|
meta.setdefault('provenance',{})
|
||
|
|
meta['provenance'].update({'import_batch':batch_id,'source_key':item['source_key'],'origin':batch['origin']})
|
||
|
|
payload['meta']=meta
|
||
|
|
before=json.dumps(original,ensure_ascii=False) if original else None
|
||
|
|
eid=save_entry(site_id,payload,existing_id=original['id'] if original else None,source='v8-import')
|
||
|
|
after=db.one('SELECT * FROM entries WHERE site_id=? AND id=?',(site_id,eid))
|
||
|
|
db.execute('''UPDATE v8_import_items SET state='applied',entry_id=?,before_json=?,after_fingerprint=? WHERE id=?''',
|
||
|
|
(eid,before,_row_hash(after),item['id']))
|
||
|
|
if original:updated+=1
|
||
|
|
else:created+=1
|
||
|
|
except Exception as exc:
|
||
|
|
msg=str(exc)[:250]
|
||
|
|
db.execute("UPDATE v8_import_items SET state='error',error=? WHERE id=?",(msg,item['id']))
|
||
|
|
failed.append({'row':item['row_number'],'message':msg})
|
||
|
|
db.execute('UPDATE v8_import_batches SET state=?,committed_at=? WHERE id=?',
|
||
|
|
('partial' if failed else 'committed',db.now(),batch_id))
|
||
|
|
# Keep unsupported Elementor/WooCommerce details in the existing release gate.
|
||
|
|
if batch['origin']=='wordpress-wxr':
|
||
|
|
for finding in json.loads(batch.get('warnings_json') or '[]')[:200]:
|
||
|
|
db.insert('''INSERT INTO cms_import_reviews(id,site_id,platform,row_number,issue,created_at)
|
||
|
|
VALUES(?,?,?,?,?,?)''',(uuid.uuid4().hex,site_id,'wordpress-wxr',int(finding.get('row',0)),
|
||
|
|
('v8:'+batch_id+': '+str(finding.get('issue','Requires review')))[:600],db.now()))
|
||
|
|
return {'batch_id':batch_id,'state':'partial' if failed else 'committed',
|
||
|
|
'created':created,'updated':updated,'skipped':skipped,'errors':failed}
|
||
|
|
|
||
|
|
def rollback(site_id,batch_id):
|
||
|
|
# Hold one write transaction across the conflict check and all reversions.
|
||
|
|
# A manual edit racing with rollback must not be accidentally overwritten.
|
||
|
|
with db.connect() as c:
|
||
|
|
c.execute('BEGIN IMMEDIATE')
|
||
|
|
batch=c.execute('SELECT * FROM v8_import_batches WHERE site_id=? AND id=?',(site_id,batch_id)).fetchone()
|
||
|
|
if not batch:raise HTTPException(404,'导入批次不存在')
|
||
|
|
if batch['state']=='rolled_back':return {'batch_id':batch_id,'state':'rolled_back','existing':True}
|
||
|
|
if batch['state'] not in ('committed','partial'):raise HTTPException(409,'批次还没有完成提交')
|
||
|
|
applied=c.execute("SELECT * FROM v8_import_items WHERE batch_id=? AND state='applied' ORDER BY row_number DESC",(batch_id,)).fetchall()
|
||
|
|
conflicts=[]
|
||
|
|
for item in applied:
|
||
|
|
current=c.execute('SELECT * FROM entries WHERE site_id=? AND id=?',(site_id,item['entry_id'])).fetchone()
|
||
|
|
if _row_hash(dict(current) if current else None)!=item['after_fingerprint']:
|
||
|
|
conflicts.append(item['route'])
|
||
|
|
if conflicts:raise HTTPException(409,{'message':'内容在导入后被修改，拒绝覆盖人工编辑','routes':conflicts})
|
||
|
|
cols=('type','title','route','slug','summary','body','cover','seo_title','seo_description',
|
||
|
|
'canonical','robots','meta_json','status','source','created_at','updated_at')
|
||
|
|
for item in applied:
|
||
|
|
if not item['before_json']:
|
||
|
|
c.execute('DELETE FROM entries WHERE site_id=? AND id=?',(site_id,item['entry_id']))
|
||
|
|
else:
|
||
|
|
before=json.loads(item['before_json'])
|
||
|
|
assignments=','.join(k+'=?' for k in cols)
|
||
|
|
c.execute(f'UPDATE entries SET {assignments} WHERE site_id=? AND id=?',
|
||
|
|
(*[before[k] for k in cols],site_id,item['entry_id']))
|
||
|
|
c.execute("UPDATE v8_import_items SET state='rolled_back' WHERE id=?",(item['id'],))
|
||
|
|
c.execute("UPDATE v8_import_batches SET state='rolled_back',rolled_back_at=? WHERE id=?",(db.now(),batch_id))
|
||
|
|
if batch['origin']=='wordpress-wxr':
|
||
|
|
c.execute("UPDATE cms_import_reviews SET state='resolved',reviewer='system',resolution='Source import batch rolled back',resolved_at=? WHERE site_id=? AND platform='wordpress-wxr' AND state='pending' AND issue LIKE ?",
|
||
|
|
(db.now(),site_id,'v8:'+batch_id+': %'))
|
||
|
|
return {'batch_id':batch_id,'state':'rolled_back','restored':len(applied)}
|
||
|
|
|
||
|
|
def parse_wxr(content):
|
||
|
|
"""Extract conservative WordPress records without writing into the site.
|
||
|
|
Unsupported builder configuration stays in source metadata for review.
|
||
|
|
"""
|
||
|
|
from defusedxml import ElementTree as ET
|
||
|
|
if len(content)>MAX_UPLOAD:raise ValueError('XML 文件超过限制')
|
||
|
|
root=ET.fromstring(content)
|
||
|
|
if root.tag!='rss':raise ValueError('这不是 WordPress WXR RSS 导出')
|
||
|
|
rows=[];ids=[];warnings=[]
|
||
|
|
for it in root.findall('./channel/item'):
|
||
|
|
get=lambda name:next((str(node.text or '') for node in it if node.tag.endswith('}'+name) or node.tag==name),'')
|
||
|
|
raw_typ=get('post_type') or 'post'
|
||
|
|
status=get('status') or 'publish'
|
||
|
|
if raw_typ in ('attachment','revision','nav_menu_item') or status not in ('publish','draft','private'):continue
|
||
|
|
if len(rows)>=MAX_ITEMS:raise ValueError('单批 WXR 最多1000条，请拆分文件')
|
||
|
|
wid=get('post_id')
|
||
|
|
title=get('title').strip() or get('post_name').strip()
|
||
|
|
if not title:continue
|
||
|
|
url=get('link')
|
||
|
|
slug=get('post_name') or re.sub(r'[^a-z0-9-]','-',title.lower())[:65] or uuid.uuid4().hex[:8]
|
||
|
|
route=urlsplit(url).path if url.startswith(('http://','https://')) else ''
|
||
|
|
if not route:route=f'/{slug}/' if raw_typ=='page' else f'/{raw_typ}/{slug}/'
|
||
|
|
metas={}
|
||
|
|
for pm in it:
|
||
|
|
if not pm.tag.endswith('}postmeta'):continue
|
||
|
|
k=v=''
|
||
|
|
for n in pm:
|
||
|
|
if n.tag.endswith('}meta_key'):k=n.text or ''
|
||
|
|
if n.tag.endswith('}meta_value'):v=n.text or ''
|
||
|
|
if k:metas[k]=v[:10000]
|
||
|
|
if any(k in metas for k in ('_elementor_data','_product_attributes')):
|
||
|
|
warnings.append({'row':len(rows)+1,'issue':'含复杂 Elementor/WooCommerce 组件，导入后需人工核对'})
|
||
|
|
rows.append({'标题':title,'路径':route,'正文':get('encoded'),
|
||
|
|
'SEO标题':metas.get('_yoast_wpseo_title') or metas.get('rank_math_title',''),
|
||
|
|
'SEO描述':metas.get('_yoast_wpseo_metadesc') or metas.get('rank_math_description',''),
|
||
|
|
'分类':raw_typ})
|
||
|
|
ids.append(f'wordpress:{wid}' if wid else f'wordpress:path:{route}')
|
||
|
|
return rows,ids,warnings
|
||
|
|
|
||
|
|
@router.post('/{site_id}/staging')
|
||
|
|
def api_stage(request:Request,site_id:str,payload:dict=Body(...)):
|
||
|
|
enterprise.require(request,site_id,'editor')
|
||
|
|
try:return stage(site_id,payload.get('rows'),payload.get('mapping'),payload.get('content_type','product'),payload.get('origin','manual-import'))
|
||
|
|
except ValueError as e:raise HTTPException(400,str(e))
|
||
|
|
|
||
|
|
@router.post('/{site_id}/staging/file')
|
||
|
|
async def api_table_file(request:Request,site_id:str,file:UploadFile=File(...),
|
||
|
|
content_type:str=Form('product')):
|
||
|
|
enterprise.require(request,site_id,'editor')
|
||
|
|
from .table_loader import parse_table
|
||
|
|
blob=await file.read(MAX_UPLOAD+1)
|
||
|
|
try:
|
||
|
|
parsed=parse_table(file.filename or '',blob)
|
||
|
|
result=stage(site_id,parsed['rows'],kind=content_type,origin='table:'+parsed['filename'][:120])
|
||
|
|
result['columns']=parsed['columns']
|
||
|
|
return result
|
||
|
|
except (ValueError,UnicodeError,json.JSONDecodeError) as e:
|
||
|
|
raise HTTPException(400,str(e))
|
||
|
|
|
||
|
|
@router.post('/{site_id}/staging/wxr')
|
||
|
|
async def api_wxr(request:Request,site_id:str,file:UploadFile=File(...)):
|
||
|
|
enterprise.require(request,site_id,'editor')
|
||
|
|
if not (file.filename or '').lower().endswith('.xml'):raise HTTPException(400,'仅支持 .xml 导出')
|
||
|
|
blob=await file.read(MAX_UPLOAD+1)
|
||
|
|
try:
|
||
|
|
rows,keys,warnings=parse_wxr(blob)
|
||
|
|
if not rows:raise ValueError('没有可导入的 WordPress 页面或文章')
|
||
|
|
result=stage(site_id,rows,{'title':'标题','route':'路径','body':'正文',
|
||
|
|
'seo_title':'SEO标题','seo_description':'SEO描述'},'mixed','wordpress-wxr',keys)
|
||
|
|
db.execute('UPDATE v8_import_batches SET warnings_json=? WHERE id=?',
|
||
|
|
(json.dumps(warnings,ensure_ascii=False),result['batch_id']))
|
||
|
|
return {**result,'warnings':warnings[:30]}
|
||
|
|
except ValueError as e:raise HTTPException(400,str(e))
|
||
|
|
|
||
|
|
@router.get('/{site_id}/staging')
|
||
|
|
def api_list(request:Request,site_id:str):
|
||
|
|
enterprise.require(request,site_id,'viewer')
|
||
|
|
return {'items':list_batches(site_id)}
|
||
|
|
|
||
|
|
@router.get('/{site_id}/staging/{batch_id}')
|
||
|
|
def api_detail(request:Request,site_id:str,batch_id:str):
|
||
|
|
enterprise.require(request,site_id,'viewer')
|
||
|
|
return batch_detail(site_id,batch_id)
|
||
|
|
|
||
|
|
@router.post('/{site_id}/staging/{batch_id}/commit')
|
||
|
|
def api_commit(request:Request,site_id:str,batch_id:str,payload:dict=Body(default_factory=dict)):
|
||
|
|
enterprise.require(request,site_id,'editor')
|
||
|
|
try:return commit(site_id,batch_id,payload.get('on_conflict','skip'))
|
||
|
|
except ValueError as e:raise HTTPException(400,str(e))
|
||
|
|
|
||
|
|
@router.post('/{site_id}/staging/{batch_id}/rollback')
|
||
|
|
def api_rollback(request:Request,site_id:str,batch_id:str):
|
||
|
|
enterprise.require(request,site_id,'owner')
|
||
|
|
return rollback(site_id,batch_id)
|