185 lines
11 KiB
Python
185 lines
11 KiB
Python
|
"""Deterministic import adapters for exported CMS JSON.
|
||
|
|
|
||
|
|
No PHP execution, no private API credential collection, no guessing missing SEO.
|
||
|
|
Every unsupported complex field is flagged for manual migration review.
|
||
|
|
"""
|
||
|
|
from __future__ import annotations
|
||
|
|
import json,re,uuid
|
||
|
|
from html import unescape
|
||
|
|
from urllib.parse import urlparse
|
||
|
|
from bs4 import BeautifulSoup
|
||
|
|
from fastapi import APIRouter,HTTPException,Request,UploadFile,File,Form,Body
|
||
|
|
from . import enterprise,importers,wordpress_compat,db
|
||
|
|
|
||
|
|
router=APIRouter(prefix='/api/migrations',tags=['CMS import'])
|
||
|
|
PLATFORMS=('wordpress-rest','woocommerce','drupal-jsonapi','shopify')
|
||
|
|
MAX_BYTES=12_000_000
|
||
|
|
MAX_ITEMS=1200
|
||
|
|
|
||
|
|
# Persist review findings: importer warnings must not vanish after the response.
|
||
|
|
def setup():
|
||
|
|
with db.connect() as conn:
|
||
|
|
conn.executescript("""CREATE TABLE IF NOT EXISTS cms_import_reviews (
|
||
|
|
id TEXT PRIMARY KEY, site_id TEXT NOT NULL REFERENCES sites(id) ON DELETE CASCADE,
|
||
|
|
platform TEXT NOT NULL, row_number INTEGER NOT NULL, issue TEXT NOT NULL,
|
||
|
|
state TEXT NOT NULL DEFAULT 'pending', reviewer TEXT NOT NULL DEFAULT '',
|
||
|
|
resolution TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL, resolved_at TEXT);
|
||
|
|
CREATE INDEX IF NOT EXISTS idx_cms_reviews_site ON cms_import_reviews(site_id,state);""")
|
||
|
|
|
||
|
|
def open_reviews(site_id):
|
||
|
|
row=db.one('SELECT COUNT(*) AS n FROM cms_import_reviews WHERE site_id=? AND state=?',
|
||
|
|
(site_id,'pending'))
|
||
|
|
return int(row['n']) if row else 0
|
||
|
|
|
||
|
|
@router.get('/sites/{site_id}/reviews')
|
||
|
|
def list_reviews(request:Request,site_id:str,state:str='pending'):
|
||
|
|
enterprise.require(request,site_id,'viewer')
|
||
|
|
if state not in ('pending','resolved','all'):raise HTTPException(400,'Invalid review state')
|
||
|
|
if state=='all':
|
||
|
|
items=db.rows('SELECT * FROM cms_import_reviews WHERE site_id=? ORDER BY created_at DESC,id DESC LIMIT 300',(site_id,))
|
||
|
|
else:
|
||
|
|
items=db.rows('SELECT * FROM cms_import_reviews WHERE site_id=? AND state=? ORDER BY created_at DESC,id DESC LIMIT 300',(site_id,state))
|
||
|
|
return {'items':items,'pending':open_reviews(site_id)}
|
||
|
|
|
||
|
|
@router.post('/sites/{site_id}/reviews/{review_id}/resolve')
|
||
|
|
def resolve_review(request:Request,site_id:str,review_id:str,payload:dict=Body(...)):
|
||
|
|
enterprise.require(request,site_id,'reviewer')
|
||
|
|
explanation=str(payload.get('resolution','')).strip()
|
||
|
|
if len(explanation)<12 or len(explanation)>2000:
|
||
|
|
raise HTTPException(400,'Resolution must explain the manual review in 12-2000 characters')
|
||
|
|
updated=db.execute("UPDATE cms_import_reviews SET state='resolved', reviewer=?, resolution=?, resolved_at=? WHERE site_id=? AND id=? AND state='pending'",
|
||
|
|
(request.state.actor,explanation,db.now(),site_id,review_id))
|
||
|
|
if not updated:raise HTTPException(404,'Review finding not found or already resolved')
|
||
|
|
return {'resolved':True,'pending':open_reviews(site_id)}
|
||
|
|
|
||
|
|
def plain(value):
|
||
|
|
if isinstance(value,dict):value=value.get('rendered',value.get('value',''))
|
||
|
|
return BeautifulSoup(str(value or ''),'html.parser').get_text(' ',strip=True)
|
||
|
|
|
||
|
|
def path(value,fallback):
|
||
|
|
if isinstance(value,dict):value=value.get('alias') or value.get('uri')
|
||
|
|
raw=str(value or '').strip()
|
||
|
|
if raw.startswith(('http://','https://')):raw=urlparse(raw).path
|
||
|
|
return raw if raw.startswith('/') else fallback
|
||
|
|
|
||
|
|
def mapped_wp(rec,kind):
|
||
|
|
seo=rec.get('yoast_head_json') or rec.get('rank_math_head_json') or {}
|
||
|
|
meta=rec.get('meta') if isinstance(rec.get('meta'),dict) else {}
|
||
|
|
acf=rec.get('acf') if isinstance(rec.get('acf'),dict) else {}
|
||
|
|
widgets=meta.get('_elementor_data')
|
||
|
|
warnings=[]
|
||
|
|
body=rec.get('content') or rec.get('description') or ''
|
||
|
|
if isinstance(body,dict):body=body.get('rendered','')
|
||
|
|
if isinstance(widgets,str) and widgets.strip():
|
||
|
|
el=wordpress_compat.elementor_to_html(widgets)
|
||
|
|
if not body and el['html']:body=el['html']
|
||
|
|
warnings+=el['warnings']+['Elementor 原始布局需人工核对']
|
||
|
|
for field,val in acf.items():
|
||
|
|
if isinstance(val,(dict,list)):
|
||
|
|
warnings.append('ACF 复杂关系字段需检查：'+field[:80])
|
||
|
|
gallery=[]
|
||
|
|
for obj in rec.get('images',[]) if isinstance(rec.get('images'),list) else []:
|
||
|
|
if isinstance(obj,dict) and obj.get('src'):gallery.append(obj['src'])
|
||
|
|
if kind=='product':
|
||
|
|
for item in rec.get('attributes',[]) if isinstance(rec.get('attributes'),list) else []:
|
||
|
|
if isinstance(item,dict) and item.get('variation'):warnings.append('WooCommerce 变体属性需要单独映射')
|
||
|
|
if rec.get('variations'):warnings.append('WooCommerce 变体/价格组合需人工校验')
|
||
|
|
title=plain(rec.get('title') or rec.get('name'))
|
||
|
|
identifier=rec.get('slug') or rec.get('id') or 'item'
|
||
|
|
fallback=f'/{kind}/{identifier}/'
|
||
|
|
route=path(rec.get('link') or rec.get('permalink'),fallback)
|
||
|
|
return {'type':kind,'title':title,'route':route,'summary':plain(rec.get('excerpt') or rec.get('short_description')),
|
||
|
|
'body':str(body),'cover':rec.get('featured_image_url','') or (gallery[0] if gallery else ''),
|
||
|
|
'seo_title':plain(seo.get('title','')),'seo_description':plain(seo.get('description','')),
|
||
|
|
'canonical':seo.get('canonical',''),
|
||
|
|
'meta':{'fields':{**{k:v for k,v in acf.items() if not isinstance(v,(dict,list))},
|
||
|
|
**({'sku':rec.get('sku'),'price':rec.get('price'),'gallery':gallery} if kind=='product' else {})},
|
||
|
|
'migration_warnings':warnings,'source_id':rec.get('id')}}
|
||
|
|
|
||
|
|
def mapped_drupal(rec):
|
||
|
|
a=rec.get('attributes') if isinstance(rec.get('attributes'),dict) else {}
|
||
|
|
typ=str(rec.get('type') or '')
|
||
|
|
kind='post' if 'article' in typ or 'news' in typ else ('product' if 'product' in typ else 'page')
|
||
|
|
title=plain(a.get('title') or a.get('name'))
|
||
|
|
raw=a.get('body') or a.get('field_description') or ''
|
||
|
|
body=raw.get('processed',raw.get('value','')) if isinstance(raw,dict) else str(raw)
|
||
|
|
u=a.get('path') or a.get('alias')
|
||
|
|
route=path(u,f'/{kind}/{rec.get("id") or "item"}/')
|
||
|
|
meta=a.get('metatag') or []
|
||
|
|
metad={x.get('attributes',{}).get('name'):x.get('attributes',{}).get('content') for x in meta if isinstance(x,dict)} if isinstance(meta,list) else {}
|
||
|
|
links=rec.get('relationships') or {}
|
||
|
|
warnings=['Drupal 实体引用关系需映射：'+str(key)[:60] for key,val in links.items() if isinstance(val,dict) and val.get('data')]
|
||
|
|
return {'type':kind,'title':title,'route':route,'body':body,'seo_title':metad.get('title') or '',
|
||
|
|
'seo_description':metad.get('description') or '',
|
||
|
|
'meta':{'migration_warnings':warnings,'source_id':rec.get('id'),'source_type':typ}}
|
||
|
|
|
||
|
|
def mapped_shopify(rec):
|
||
|
|
handle=rec.get('handle') or rec.get('id') or 'product'
|
||
|
|
images=rec.get('images',[])
|
||
|
|
gallery=[x.get('src') for x in images if isinstance(x,dict) and x.get('src')]
|
||
|
|
warnings=[]
|
||
|
|
variants=rec.get('variants',[])
|
||
|
|
if len(variants)>1:warnings.append('Shopify 多变体/库存/价格需人工复核')
|
||
|
|
if rec.get('options'):warnings.append('Shopify options 与变体关系需审核')
|
||
|
|
return {'type':'product','title':plain(rec.get('title')),'route':f'/products/{handle}/',
|
||
|
|
'summary':'','body':str(rec.get('body_html') or ''),'cover':gallery[0] if gallery else '',
|
||
|
|
'seo_title':rec.get('metafields_global_title_tag') or '',
|
||
|
|
'seo_description':rec.get('metafields_global_description_tag') or '',
|
||
|
|
'meta':{'fields':{'sku':variants[0].get('sku','') if variants else '',
|
||
|
|
'price':variants[0].get('price','') if variants else '',
|
||
|
|
'gallery':gallery,'brand':rec.get('vendor',''),'category':rec.get('product_type','')},
|
||
|
|
'migration_warnings':warnings,'source_id':rec.get('id')}}
|
||
|
|
|
||
|
|
def parse_export(platform,blob):
|
||
|
|
if platform not in PLATFORMS:raise ValueError('Unsupported CMS adapter')
|
||
|
|
if len(blob)>MAX_BYTES:raise ValueError('Import archive too large')
|
||
|
|
source=json.loads(blob.decode('utf-8-sig'))
|
||
|
|
if platform=='drupal-jsonapi':
|
||
|
|
records=source.get('data',[]) if isinstance(source,dict) else source
|
||
|
|
elif platform=='shopify':
|
||
|
|
records=source.get('products',[]) if isinstance(source,dict) else source
|
||
|
|
else:
|
||
|
|
records=source.get('products',source.get('entries',source.get('items',[]))) if isinstance(source,dict) else source
|
||
|
|
if not isinstance(records,list) or len(records)>MAX_ITEMS:raise ValueError('Expected <=1200 array items')
|
||
|
|
result=[];errors=[];warnings=[];routes=set()
|
||
|
|
for index,raw in enumerate(records,1):
|
||
|
|
try:
|
||
|
|
if not isinstance(raw,dict):raise ValueError('Item not an object')
|
||
|
|
if platform=='drupal-jsonapi':entry=mapped_drupal(raw)
|
||
|
|
elif platform=='shopify':entry=mapped_shopify(raw)
|
||
|
|
else:entry=mapped_wp(raw,'product' if platform=='woocommerce' else ('page' if raw.get('type')=='page' else 'post'))
|
||
|
|
if not entry['title']:raise ValueError('Missing title')
|
||
|
|
from .core import route_path
|
||
|
|
entry['route']=route_path(entry['route'])
|
||
|
|
if entry['route'] in routes:raise ValueError('Duplicate route '+entry['route'])
|
||
|
|
routes.add(entry['route'])
|
||
|
|
for warning in entry.get('meta',{}).get('migration_warnings',[]):warnings.append({'row':index,'issue':warning})
|
||
|
|
result.append(entry)
|
||
|
|
except (ValueError,TypeError,KeyError) as e:errors.append({'row':index,'error':str(e)[:180]})
|
||
|
|
return {'platform':platform,'input':len(records),'mapped':len(result),'errors':errors,'warnings':warnings[:250],'preview':result[:8],'entries':result}
|
||
|
|
|
||
|
|
@router.post('/sites/{site_id}/preview')
|
||
|
|
async def preview(request:Request,site_id:str,platform:str=Form(...),file:UploadFile=File(...)):
|
||
|
|
enterprise.require(request,site_id,'editor')
|
||
|
|
try:
|
||
|
|
blob=await file.read(MAX_BYTES+1)
|
||
|
|
result=parse_export(platform,blob)
|
||
|
|
except (ValueError,UnicodeError,json.JSONDecodeError) as exc:raise HTTPException(400,str(exc))
|
||
|
|
return {k:v for k,v in result.items() if k!='entries'}
|
||
|
|
|
||
|
|
@router.post('/sites/{site_id}/import')
|
||
|
|
async def upload(request:Request,site_id:str,platform:str=Form(...),conflict:str=Form('skip'),file:UploadFile=File(...)):
|
||
|
|
enterprise.require(request,site_id,'editor')
|
||
|
|
if conflict not in ('skip','update'):raise HTTPException(400,'Invalid conflict policy')
|
||
|
|
try:result=parse_export(platform,await file.read(MAX_BYTES+1))
|
||
|
|
except (ValueError,UnicodeError,json.JSONDecodeError) as exc:raise HTTPException(400,str(exc))
|
||
|
|
if result['errors']:raise HTTPException(409,'导入数据存在错误；请先预览并修正')
|
||
|
|
committed=importers.import_records(site_id,result['entries'],platform,on_conflict=conflict)
|
||
|
|
if result['warnings']:
|
||
|
|
with db.connect() as c:
|
||
|
|
for warning in result['warnings']:
|
||
|
|
c.execute('INSERT INTO cms_import_reviews(id,site_id,platform,row_number,issue,state,created_at) VALUES (?,?,?,?,?,?,?)',
|
||
|
|
(uuid.uuid4().hex,site_id,platform,warning['row'],warning['issue'],'pending',db.now()))
|
||
|
|
return {'result':committed,'migration_warnings':result['warnings'],
|
||
|
|
'manual_review_required':bool(result['warnings']),'pending_review_count':open_reviews(site_id)}
|