"""Deterministic import adapters for exported CMS JSON. No PHP execution, no private API credential collection, no guessing missing SEO. Every unsupported complex field is flagged for manual migration review. """ from __future__ import annotations import json,re,uuid from html import unescape from urllib.parse import urlparse from bs4 import BeautifulSoup from fastapi import APIRouter,HTTPException,Request,UploadFile,File,Form,Body from . import enterprise,importers,wordpress_compat,db router=APIRouter(prefix='/api/migrations',tags=['CMS import']) PLATFORMS=('wordpress-rest','woocommerce','drupal-jsonapi','shopify') MAX_BYTES=12_000_000 MAX_ITEMS=1200 # Persist review findings: importer warnings must not vanish after the response. def setup(): with db.connect() as conn: conn.executescript("""CREATE TABLE IF NOT EXISTS cms_import_reviews ( id TEXT PRIMARY KEY, site_id TEXT NOT NULL REFERENCES sites(id) ON DELETE CASCADE, platform TEXT NOT NULL, row_number INTEGER NOT NULL, issue TEXT NOT NULL, state TEXT NOT NULL DEFAULT 'pending', reviewer TEXT NOT NULL DEFAULT '', resolution TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL, resolved_at TEXT); CREATE INDEX IF NOT EXISTS idx_cms_reviews_site ON cms_import_reviews(site_id,state);""") def open_reviews(site_id): row=db.one('SELECT COUNT(*) AS n FROM cms_import_reviews WHERE site_id=? AND state=?', (site_id,'pending')) return int(row['n']) if row else 0 @router.get('/sites/{site_id}/reviews') def list_reviews(request:Request,site_id:str,state:str='pending'): enterprise.require(request,site_id,'viewer') if state not in ('pending','resolved','all'):raise HTTPException(400,'Invalid review state') if state=='all': items=db.rows('SELECT * FROM cms_import_reviews WHERE site_id=? ORDER BY created_at DESC,id DESC LIMIT 300',(site_id,)) else: items=db.rows('SELECT * FROM cms_import_reviews WHERE site_id=? AND state=? ORDER BY created_at DESC,id DESC LIMIT 300',(site_id,state)) return {'items':items,'pending':open_reviews(site_id)} @router.post('/sites/{site_id}/reviews/{review_id}/resolve') def resolve_review(request:Request,site_id:str,review_id:str,payload:dict=Body(...)): enterprise.require(request,site_id,'reviewer') explanation=str(payload.get('resolution','')).strip() if len(explanation)<12 or len(explanation)>2000: raise HTTPException(400,'Resolution must explain the manual review in 12-2000 characters') updated=db.execute("UPDATE cms_import_reviews SET state='resolved', reviewer=?, resolution=?, resolved_at=? WHERE site_id=? AND id=? AND state='pending'", (request.state.actor,explanation,db.now(),site_id,review_id)) if not updated:raise HTTPException(404,'Review finding not found or already resolved') return {'resolved':True,'pending':open_reviews(site_id)} def plain(value): if isinstance(value,dict):value=value.get('rendered',value.get('value','')) return BeautifulSoup(str(value or ''),'html.parser').get_text(' ',strip=True) def path(value,fallback): if isinstance(value,dict):value=value.get('alias') or value.get('uri') raw=str(value or '').strip() if raw.startswith(('http://','https://')):raw=urlparse(raw).path return raw if raw.startswith('/') else fallback def mapped_wp(rec,kind): seo=rec.get('yoast_head_json') or rec.get('rank_math_head_json') or {} meta=rec.get('meta') if isinstance(rec.get('meta'),dict) else {} acf=rec.get('acf') if isinstance(rec.get('acf'),dict) else {} widgets=meta.get('_elementor_data') warnings=[] body=rec.get('content') or rec.get('description') or '' if isinstance(body,dict):body=body.get('rendered','') if isinstance(widgets,str) and widgets.strip(): el=wordpress_compat.elementor_to_html(widgets) if not body and el['html']:body=el['html'] warnings+=el['warnings']+['Elementor 原始布局需人工核对'] for field,val in acf.items(): if isinstance(val,(dict,list)): warnings.append('ACF 复杂关系字段需检查:'+field[:80]) gallery=[] for obj in rec.get('images',[]) if isinstance(rec.get('images'),list) else []: if isinstance(obj,dict) and obj.get('src'):gallery.append(obj['src']) if kind=='product': for item in rec.get('attributes',[]) if isinstance(rec.get('attributes'),list) else []: if isinstance(item,dict) and item.get('variation'):warnings.append('WooCommerce 变体属性需要单独映射') if rec.get('variations'):warnings.append('WooCommerce 变体/价格组合需人工校验') title=plain(rec.get('title') or rec.get('name')) identifier=rec.get('slug') or rec.get('id') or 'item' fallback=f'/{kind}/{identifier}/' route=path(rec.get('link') or rec.get('permalink'),fallback) return {'type':kind,'title':title,'route':route,'summary':plain(rec.get('excerpt') or rec.get('short_description')), 'body':str(body),'cover':rec.get('featured_image_url','') or (gallery[0] if gallery else ''), 'seo_title':plain(seo.get('title','')),'seo_description':plain(seo.get('description','')), 'canonical':seo.get('canonical',''), 'meta':{'fields':{**{k:v for k,v in acf.items() if not isinstance(v,(dict,list))}, **({'sku':rec.get('sku'),'price':rec.get('price'),'gallery':gallery} if kind=='product' else {})}, 'migration_warnings':warnings,'source_id':rec.get('id')}} def mapped_drupal(rec): a=rec.get('attributes') if isinstance(rec.get('attributes'),dict) else {} typ=str(rec.get('type') or '') kind='post' if 'article' in typ or 'news' in typ else ('product' if 'product' in typ else 'page') title=plain(a.get('title') or a.get('name')) raw=a.get('body') or a.get('field_description') or '' body=raw.get('processed',raw.get('value','')) if isinstance(raw,dict) else str(raw) u=a.get('path') or a.get('alias') route=path(u,f'/{kind}/{rec.get("id") or "item"}/') meta=a.get('metatag') or [] metad={x.get('attributes',{}).get('name'):x.get('attributes',{}).get('content') for x in meta if isinstance(x,dict)} if isinstance(meta,list) else {} links=rec.get('relationships') or {} warnings=['Drupal 实体引用关系需映射:'+str(key)[:60] for key,val in links.items() if isinstance(val,dict) and val.get('data')] return {'type':kind,'title':title,'route':route,'body':body,'seo_title':metad.get('title') or '', 'seo_description':metad.get('description') or '', 'meta':{'migration_warnings':warnings,'source_id':rec.get('id'),'source_type':typ}} def mapped_shopify(rec): handle=rec.get('handle') or rec.get('id') or 'product' images=rec.get('images',[]) gallery=[x.get('src') for x in images if isinstance(x,dict) and x.get('src')] warnings=[] variants=rec.get('variants',[]) if len(variants)>1:warnings.append('Shopify 多变体/库存/价格需人工复核') if rec.get('options'):warnings.append('Shopify options 与变体关系需审核') return {'type':'product','title':plain(rec.get('title')),'route':f'/products/{handle}/', 'summary':'','body':str(rec.get('body_html') or ''),'cover':gallery[0] if gallery else '', 'seo_title':rec.get('metafields_global_title_tag') or '', 'seo_description':rec.get('metafields_global_description_tag') or '', 'meta':{'fields':{'sku':variants[0].get('sku','') if variants else '', 'price':variants[0].get('price','') if variants else '', 'gallery':gallery,'brand':rec.get('vendor',''),'category':rec.get('product_type','')}, 'migration_warnings':warnings,'source_id':rec.get('id')}} def parse_export(platform,blob): if platform not in PLATFORMS:raise ValueError('Unsupported CMS adapter') if len(blob)>MAX_BYTES:raise ValueError('Import archive too large') source=json.loads(blob.decode('utf-8-sig')) if platform=='drupal-jsonapi': records=source.get('data',[]) if isinstance(source,dict) else source elif platform=='shopify': records=source.get('products',[]) if isinstance(source,dict) else source else: records=source.get('products',source.get('entries',source.get('items',[]))) if isinstance(source,dict) else source if not isinstance(records,list) or len(records)>MAX_ITEMS:raise ValueError('Expected <=1200 array items') result=[];errors=[];warnings=[];routes=set() for index,raw in enumerate(records,1): try: if not isinstance(raw,dict):raise ValueError('Item not an object') if platform=='drupal-jsonapi':entry=mapped_drupal(raw) elif platform=='shopify':entry=mapped_shopify(raw) else:entry=mapped_wp(raw,'product' if platform=='woocommerce' else ('page' if raw.get('type')=='page' else 'post')) if not entry['title']:raise ValueError('Missing title') from .core import route_path entry['route']=route_path(entry['route']) if entry['route'] in routes:raise ValueError('Duplicate route '+entry['route']) routes.add(entry['route']) for warning in entry.get('meta',{}).get('migration_warnings',[]):warnings.append({'row':index,'issue':warning}) result.append(entry) except (ValueError,TypeError,KeyError) as e:errors.append({'row':index,'error':str(e)[:180]}) return {'platform':platform,'input':len(records),'mapped':len(result),'errors':errors,'warnings':warnings[:250],'preview':result[:8],'entries':result} @router.post('/sites/{site_id}/preview') async def preview(request:Request,site_id:str,platform:str=Form(...),file:UploadFile=File(...)): enterprise.require(request,site_id,'editor') try: blob=await file.read(MAX_BYTES+1) result=parse_export(platform,blob) except (ValueError,UnicodeError,json.JSONDecodeError) as exc:raise HTTPException(400,str(exc)) return {k:v for k,v in result.items() if k!='entries'} @router.post('/sites/{site_id}/import') async def upload(request:Request,site_id:str,platform:str=Form(...),conflict:str=Form('skip'),file:UploadFile=File(...)): enterprise.require(request,site_id,'editor') if conflict not in ('skip','update'):raise HTTPException(400,'Invalid conflict policy') try:result=parse_export(platform,await file.read(MAX_BYTES+1)) except (ValueError,UnicodeError,json.JSONDecodeError) as exc:raise HTTPException(400,str(exc)) if result['errors']:raise HTTPException(409,'导入数据存在错误;请先预览并修正') committed=importers.import_records(site_id,result['entries'],platform,on_conflict=conflict) if result['warnings']: with db.connect() as c: for warning in result['warnings']: c.execute('INSERT INTO cms_import_reviews(id,site_id,platform,row_number,issue,state,created_at) VALUES (?,?,?,?,?,?,?)', (uuid.uuid4().hex,site_id,platform,warning['row'],warning['issue'],'pending',db.now())) return {'result':committed,'migration_warnings':result['warnings'], 'manual_review_required':bool(result['warnings']),'pending_review_count':open_reviews(site_id)}