chore: baseline V8.10.0-dev.1 with v8_workflows type-mapping fix and trade-pro spec
- SiteForge Enterprise V8.10.0-dev.1 upstream source (210 tests) - Fix: v8_workflows._mapped_input now reads both 类型/分类 columns so WXR imports keep post/product types instead of collapsing to page - Docs: trade-pro theme port spec (R3) and implementation plan Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -0,0 +1,245 @@
|
||||
"""Offline, reproducible website migration inventory and built-artifact comparison.
|
||||
|
||||
No network calls. The source manifest is treated as untrusted input and never as
|
||||
an instruction. Unsupported dynamic URLs are reported instead of rewritten.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import csv, hashlib, io, json, re
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
from bs4 import BeautifulSoup
|
||||
from defusedxml import ElementTree as ET
|
||||
from fastapi import APIRouter, Body, File, HTTPException, Request, UploadFile
|
||||
from . import db, enterprise, builder
|
||||
from .core import route_path
|
||||
|
||||
router=APIRouter(prefix='/api/audit',tags=['Migration Audit V4'])
|
||||
MAX_BASELINE=20_000
|
||||
|
||||
def setup():
|
||||
with db.connect() as c:
|
||||
c.executescript('''CREATE TABLE IF NOT EXISTS source_audit_fields(
|
||||
site_id TEXT NOT NULL, route TEXT NOT NULL,
|
||||
robots TEXT NOT NULL DEFAULT '', h1 TEXT NOT NULL DEFAULT '',
|
||||
original_url TEXT NOT NULL DEFAULT '',
|
||||
PRIMARY KEY(site_id,route),
|
||||
FOREIGN KEY(site_id) REFERENCES sites(id) ON DELETE CASCADE);
|
||||
CREATE TABLE IF NOT EXISTS source_audit_assets(
|
||||
site_id TEXT NOT NULL, url TEXT NOT NULL,
|
||||
source_route TEXT NOT NULL DEFAULT '',
|
||||
PRIMARY KEY(site_id,url,source_route),
|
||||
FOREIGN KEY(site_id) REFERENCES sites(id) ON DELETE CASCADE);
|
||||
CREATE TABLE IF NOT EXISTS source_audit_exceptions(
|
||||
site_id TEXT NOT NULL, build_id TEXT NOT NULL, issue_id TEXT NOT NULL,
|
||||
reviewer TEXT NOT NULL, explanation TEXT NOT NULL, decided_at TEXT NOT NULL,
|
||||
PRIMARY KEY(site_id,build_id,issue_id));
|
||||
''')
|
||||
|
||||
def _route(value:str):
|
||||
text=str(value or '').strip()
|
||||
if not text:raise ValueError('缺少页面路径')
|
||||
u=urlsplit(text)
|
||||
if u.query:raise ValueError('需要单独处理带查询参数的动态 URL')
|
||||
if u.scheme and u.scheme not in ('https','http'):raise ValueError('不支持的 URL 协议')
|
||||
return route_path(u.path or '/')
|
||||
|
||||
def _str(row,*keys):
|
||||
for key in keys:
|
||||
if row.get(key) is not None and str(row[key]).strip():return str(row[key]).strip()
|
||||
return ''
|
||||
|
||||
def _normalize(row):
|
||||
original=_str(row,'url','loc','original_url','link','route','path')
|
||||
route=_route(original)
|
||||
status=int(row.get('status') or 200)
|
||||
if status not in (200,301,302,307,308,404,410):raise ValueError('不支持的 HTTP 状态')
|
||||
return {'route':route,'original_url':original,'seo_title':_str(row,'seo_title','title','meta_title'),
|
||||
'description':_str(row,'description','seo_description','meta_description'),
|
||||
'canonical':_str(row,'canonical'),'robots':_str(row,'robots'),
|
||||
'h1':_str(row,'h1'),'status':status}
|
||||
|
||||
def _wp_item(element):
|
||||
def get(local,default=''):
|
||||
for ch in element:
|
||||
if ch.tag.rsplit('}',1)[-1]==local:return (ch.text or default)
|
||||
return default
|
||||
def get_content(local):
|
||||
for ch in element:
|
||||
if ch.tag.rsplit('}',1)[-1]==local:return ch.text or ''
|
||||
return ''
|
||||
meta={}
|
||||
for child in element:
|
||||
if child.tag.rsplit('}',1)[-1]=='postmeta':
|
||||
key=value=''
|
||||
for part in child:
|
||||
if part.tag.endswith('meta_key'):key=part.text or ''
|
||||
elif part.tag.endswith('meta_value'):value=part.text or ''
|
||||
if key:meta[key]=value
|
||||
return (get('post_type','post'),get('status','publish'),get('link'),get('title'),get_content('encoded'),meta)
|
||||
|
||||
def from_file(filename:str,blob:bytes):
|
||||
if len(blob)>16_000_000:raise ValueError('源站清单不能超过 16 MB')
|
||||
name=filename.lower();rows=[];assets=[];warnings=[]
|
||||
if name.endswith('.csv'):
|
||||
rows=list(csv.DictReader(io.StringIO(blob.decode('utf-8-sig'))))
|
||||
elif name.endswith('.json'):
|
||||
data=json.loads(blob.decode('utf-8-sig'))
|
||||
rows=data.get('urls',data.get('pages',[])) if isinstance(data,dict) else data
|
||||
if not isinstance(rows,list):raise ValueError('JSON 应包含 pages/urls 数组')
|
||||
elif name.endswith('.xml'):
|
||||
root=ET.fromstring(blob)
|
||||
root_name=root.tag.rsplit('}',1)[-1]
|
||||
if root_name=='urlset':
|
||||
rows=[{'url':next((c.text or '' for c in node if c.tag.rsplit('}',1)[-1]=='loc'),'')} for node in root]
|
||||
elif root_name=='rss':
|
||||
for item in root.findall('./channel/item'):
|
||||
typ,status,url,title,body,meta=_wp_item(item)
|
||||
if typ=='attachment':
|
||||
if url:assets.append((url,''))
|
||||
continue
|
||||
if status!='publish' or typ in ('revision','nav_menu_item'):continue
|
||||
if not url:
|
||||
warnings.append('WXR 某条记录缺少真实链接，无法推断原路径：'+title[:70]);continue
|
||||
rows.append({'url':url,'title':meta.get('_yoast_wpseo_title') or meta.get('rank_math_title') or title,
|
||||
'description':meta.get('_yoast_wpseo_metadesc') or meta.get('rank_math_description') or '',
|
||||
'canonical':meta.get('_yoast_wpseo_canonical') or meta.get('rank_math_canonical_url') or '',
|
||||
'robots':'noindex,follow' if meta.get('_yoast_wpseo_meta-robots-noindex')=='1' else '',
|
||||
'h1':title})
|
||||
soup=BeautifulSoup(body,'html.parser')
|
||||
for img in soup.select('img[src]'):
|
||||
assets.append((img.get('src',''),url))
|
||||
else:raise ValueError('XML 需要 WordPress WXR 或 sitemap.xml')
|
||||
else:raise ValueError('仅支持 .csv / .json / WordPress WXR.xml / sitemap.xml')
|
||||
if not 1<=len(rows)<=MAX_BASELINE:raise ValueError('URL 清单需要1至20000条')
|
||||
out=[];errors=[];seen=set()
|
||||
for i,row in enumerate(rows,1):
|
||||
try:
|
||||
if not isinstance(row,dict):raise ValueError('记录不是对象')
|
||||
normalized=_normalize(row)
|
||||
if normalized['route'] in seen:raise ValueError('原路径重复')
|
||||
seen.add(normalized['route']);out.append(normalized)
|
||||
except Exception as exc:errors.append({'row':i,'message':str(exc)[:180]})
|
||||
return {'records':out,'assets':assets,'errors':errors,'warnings':warnings,'total':len(rows)}
|
||||
|
||||
def save(site_id,manifest,replace=True):
|
||||
if manifest['errors']:raise ValueError('URL 清单存在格式错误，请修正后再导入')
|
||||
with db.connect() as c:
|
||||
if replace:
|
||||
c.execute('DELETE FROM migration_baseline WHERE site_id=?',(site_id,))
|
||||
c.execute('DELETE FROM source_audit_fields WHERE site_id=?',(site_id,))
|
||||
c.execute('DELETE FROM source_audit_assets WHERE site_id=?',(site_id,))
|
||||
for row in manifest['records']:
|
||||
c.execute('''INSERT INTO migration_baseline(site_id,route,seo_title,description,canonical,status)
|
||||
VALUES(?,?,?,?,?,?) ON CONFLICT(site_id,route) DO UPDATE SET
|
||||
seo_title=excluded.seo_title,description=excluded.description,canonical=excluded.canonical,status=excluded.status''',
|
||||
(site_id,row['route'],row['seo_title'],row['description'],row['canonical'],row['status']))
|
||||
c.execute('''INSERT INTO source_audit_fields(site_id,route,robots,h1,original_url) VALUES(?,?,?,?,?)
|
||||
ON CONFLICT(site_id,route) DO UPDATE SET robots=excluded.robots,h1=excluded.h1,original_url=excluded.original_url''',
|
||||
(site_id,row['route'],row['robots'],row['h1'],row['original_url']))
|
||||
for url,origin in manifest['assets'][:MAX_BASELINE]:
|
||||
if not url or len(url)>1000:continue
|
||||
c.execute('INSERT OR IGNORE INTO source_audit_assets VALUES(?,?,?)',(site_id,url,origin[:500]))
|
||||
return {'imported':len(manifest['records']),'asset_references':len(manifest['assets']),'warnings':manifest['warnings']}
|
||||
|
||||
def _tag(soup,name,attrs=None):
|
||||
node=soup.find(name,attrs=attrs or {})
|
||||
return node.get('content','').strip() if node and name=='meta' else node.get_text(' ',strip=True) if node else ''
|
||||
|
||||
def audit(site_id,build_id):
|
||||
# Actual generated bytes, not source entry metadata, are the authority.
|
||||
root=(db.DATA/'builds'/site_id/build_id/'dist').resolve()
|
||||
if not root.is_dir() or not db.one('SELECT id FROM builds WHERE id=? AND site_id=?',(build_id,site_id)):
|
||||
raise ValueError('构建版本不存在')
|
||||
baseline=db.rows('''SELECT b.*,x.robots,x.h1,x.original_url FROM migration_baseline b
|
||||
LEFT JOIN source_audit_fields x ON b.site_id=x.site_id AND b.route=x.route
|
||||
WHERE b.site_id=? ORDER BY b.route''',(site_id,))
|
||||
if not baseline:return {'baseline_count':0,'errors':1,'warnings':0,'passed':False,
|
||||
'issues':[{'severity':'error','field':'baseline','message':'未导入源站 URL 清单'}]}
|
||||
redirects={r['old_path']:(r['new_path'],r['status_code']) for r in db.rows('SELECT old_path,new_path,status_code FROM redirects WHERE site_id=?',(site_id,))}
|
||||
issues=[];matched=0
|
||||
for old in baseline:
|
||||
route=old['route'];expect_status=old['status']; target=builder.output_path(root,route)
|
||||
if expect_status in (301,302,307,308,404,410):
|
||||
if expect_status in (301,302,307,308):
|
||||
if route not in redirects:
|
||||
issues.append({'route':route,'field':'redirect','severity':'error','message':'历史重定向未配置'})
|
||||
elif redirects[route][1]!=expect_status:
|
||||
issues.append({'route':route,'field':'redirect_status','severity':'error','message':'重定向状态码与旧站不一致'})
|
||||
elif target.is_file():
|
||||
issues.append({'route':route,'field':'http_status','severity':'error','message':'旧站的不存在/已删除页面被错误地生成了内容'})
|
||||
continue
|
||||
if not target.is_file():
|
||||
issues.append({'route':route,'field':'url','severity':'error','message':'旧站正常页面在生成网站中缺失'})
|
||||
continue
|
||||
matched+=1
|
||||
soup=BeautifulSoup(target.read_text(encoding='utf-8',errors='replace'),'html.parser')
|
||||
title=_tag(soup,'title')
|
||||
desc_node=soup.find('meta',attrs={'name':re.compile('^description$',re.I)})
|
||||
description=(desc_node.get('content','') if desc_node else '').strip()
|
||||
canonical_node=soup.find('link',rel=lambda value:value and 'canonical' in value)
|
||||
canonical=(canonical_node.get('href','') if canonical_node else '').strip()
|
||||
robot_node=soup.find('meta',attrs={'name':re.compile('^robots$',re.I)})
|
||||
robots=(robot_node.get('content','') if robot_node else '').strip().lower()
|
||||
h1=_tag(soup,'h1')
|
||||
for source,current,label in [('seo_title',title,'title'),('description',description,'description'),('canonical',canonical,'canonical'),('robots',robots,'robots'),('h1',h1,'h1')]:
|
||||
if old.get(source) and ' '.join(str(old[source]).split())!=' '.join(str(current).split()):
|
||||
issues.append({'route':route,'field':label,'severity':'warning','message':'旧新页面字段不一致',
|
||||
'before':str(old[source])[:160],'after':str(current)[:160]})
|
||||
other=__import__('app.quality',fromlist=['report']).report(site_id,build_id)
|
||||
for q in other['issues']:
|
||||
if q['severity']=='error':issues.append({'route':q['file'],'field':q['type'],'severity':'error','message':str(q.get('target','构建质量问题'))[:180]})
|
||||
decisions={x['issue_id']:x for x in db.rows('SELECT * FROM source_audit_exceptions WHERE site_id=? AND build_id=?',(site_id,build_id))}
|
||||
for issue in issues:
|
||||
token=json.dumps([issue.get(k,'') for k in ('route','field','message','before','after')],ensure_ascii=False,sort_keys=True)
|
||||
issue['issue_id']=hashlib.sha256(token.encode()).hexdigest()[:24]
|
||||
approved=decisions.get(issue['issue_id'])
|
||||
issue['reviewed']=bool(approved) if issue['severity']=='warning' else False
|
||||
if approved:
|
||||
issue['reviewer']=approved['reviewer']
|
||||
issue['review_reason']=approved['explanation']
|
||||
outstanding=sum(i['severity']=='warning' and not i['reviewed'] for i in issues)
|
||||
return {'baseline_count':len(baseline),'matched_pages':matched,
|
||||
'errors':sum(x['severity']=='error' for x in issues),
|
||||
'warnings':sum(x['severity']=='warning' for x in issues),
|
||||
'passed':not any(x['severity']=='error' for x in issues),'unreviewed_warnings':outstanding,
|
||||
'quality':{'files':other['files'],'errors':other['errors'],'warnings':other['warnings']},
|
||||
'issues':issues[:2000]}
|
||||
|
||||
@router.post('/sites/{site_id}/import-baseline')
|
||||
async def import_baseline(request:Request,site_id:str,file:UploadFile=File(...)):
|
||||
enterprise.require(request,site_id,'editor')
|
||||
data=await file.read(16_000_001)
|
||||
try:
|
||||
manifest=from_file(file.filename or '',data)
|
||||
if manifest['errors']:
|
||||
return {'imported':0,'errors':manifest['errors'][:200],'total':manifest['total'],'warnings':manifest['warnings']}
|
||||
return save(site_id,manifest)
|
||||
except (ValueError,UnicodeDecodeError) as exc:raise HTTPException(400,str(exc))
|
||||
|
||||
@router.get('/sites/{site_id}/builds/{build_id}/report')
|
||||
def report(request:Request,site_id:str,build_id:str):
|
||||
enterprise.require(request,site_id,'viewer')
|
||||
try:return audit(site_id,build_id)
|
||||
except ValueError as exc:raise HTTPException(404,str(exc))
|
||||
|
||||
@router.post('/sites/{site_id}/builds/{build_id}/review-warning')
|
||||
def approve_warning(request:Request,site_id:str,build_id:str,payload:dict=Body(...)):
|
||||
enterprise.require(request,site_id,'reviewer')
|
||||
issue_id=str(payload.get('issue_id',''))
|
||||
reason=str(payload.get('reason','')).strip()
|
||||
if len(reason)<10 or len(reason)>2000:raise HTTPException(400,'请填写至少10个字符的复核原因及处理依据')
|
||||
evidence=audit(site_id,build_id)
|
||||
issue=next((x for x in evidence['issues'] if x['issue_id']==issue_id and x['severity']=='warning'),None)
|
||||
if not issue:raise HTTPException(404,'只能复核此构建版本当前存在的警告')
|
||||
db.insert('INSERT INTO source_audit_exceptions(site_id,build_id,issue_id,reviewer,explanation,decided_at) VALUES(?,?,?,?,?,?) ON CONFLICT(site_id,build_id,issue_id) DO UPDATE SET reviewer=excluded.reviewer,explanation=excluded.explanation,decided_at=excluded.decided_at',
|
||||
(site_id,build_id,issue_id,request.state.actor,reason,db.now()))
|
||||
return {'approved':True,'issue_id':issue_id,'reviewer':request.state.actor}
|
||||
|
||||
@router.get('/sites/{site_id}/builds/{build_id}/report.csv')
|
||||
def report_csv(request:Request,site_id:str,build_id:str):
|
||||
from fastapi.responses import Response
|
||||
enterprise.require(request,site_id,'viewer')
|
||||
result=audit(site_id,build_id)
|
||||
dest=io.StringIO();writer=csv.DictWriter(dest,fieldnames=['severity','route','field','message','before','after'],extrasaction='ignore')
|
||||
writer.writeheader();writer.writerows(result['issues'])
|
||||
return Response('\ufeff'+dest.getvalue(),media_type='text/csv',headers={'Content-Disposition':'attachment; filename="migration-audit.csv"'})
|
||||
Reference in New Issue
Block a user