246 lines
15 KiB
Python
246 lines
15 KiB
Python
|
"""Offline, reproducible website migration inventory and built-artifact comparison.
|
||
|
|
|
||
|
|
No network calls. The source manifest is treated as untrusted input and never as
|
||
|
|
an instruction. Unsupported dynamic URLs are reported instead of rewritten.
|
||
|
|
"""
|
||
|
|
from __future__ import annotations
|
||
|
|
import csv, hashlib, io, json, re
|
||
|
|
from pathlib import Path
|
||
|
|
from urllib.parse import urlsplit
|
||
|
|
from bs4 import BeautifulSoup
|
||
|
|
from defusedxml import ElementTree as ET
|
||
|
|
from fastapi import APIRouter, Body, File, HTTPException, Request, UploadFile
|
||
|
|
from . import db, enterprise, builder
|
||
|
|
from .core import route_path
|
||
|
|
|
||
|
|
router=APIRouter(prefix='/api/audit',tags=['Migration Audit V4'])
|
||
|
|
MAX_BASELINE=20_000
|
||
|
|
|
||
|
|
def setup():
|
||
|
|
with db.connect() as c:
|
||
|
|
c.executescript('''CREATE TABLE IF NOT EXISTS source_audit_fields(
|
||
|
|
site_id TEXT NOT NULL, route TEXT NOT NULL,
|
||
|
|
robots TEXT NOT NULL DEFAULT '', h1 TEXT NOT NULL DEFAULT '',
|
||
|
|
original_url TEXT NOT NULL DEFAULT '',
|
||
|
|
PRIMARY KEY(site_id,route),
|
||
|
|
FOREIGN KEY(site_id) REFERENCES sites(id) ON DELETE CASCADE);
|
||
|
|
CREATE TABLE IF NOT EXISTS source_audit_assets(
|
||
|
|
site_id TEXT NOT NULL, url TEXT NOT NULL,
|
||
|
|
source_route TEXT NOT NULL DEFAULT '',
|
||
|
|
PRIMARY KEY(site_id,url,source_route),
|
||
|
|
FOREIGN KEY(site_id) REFERENCES sites(id) ON DELETE CASCADE);
|
||
|
|
CREATE TABLE IF NOT EXISTS source_audit_exceptions(
|
||
|
|
site_id TEXT NOT NULL, build_id TEXT NOT NULL, issue_id TEXT NOT NULL,
|
||
|
|
reviewer TEXT NOT NULL, explanation TEXT NOT NULL, decided_at TEXT NOT NULL,
|
||
|
|
PRIMARY KEY(site_id,build_id,issue_id));
|
||
|
|
''')
|
||
|
|
|
||
|
|
def _route(value:str):
|
||
|
|
text=str(value or '').strip()
|
||
|
|
if not text:raise ValueError('缺少页面路径')
|
||
|
|
u=urlsplit(text)
|
||
|
|
if u.query:raise ValueError('需要单独处理带查询参数的动态 URL')
|
||
|
|
if u.scheme and u.scheme not in ('https','http'):raise ValueError('不支持的 URL 协议')
|
||
|
|
return route_path(u.path or '/')
|
||
|
|
|
||
|
|
def _str(row,*keys):
|
||
|
|
for key in keys:
|
||
|
|
if row.get(key) is not None and str(row[key]).strip():return str(row[key]).strip()
|
||
|
|
return ''
|
||
|
|
|
||
|
|
def _normalize(row):
|
||
|
|
original=_str(row,'url','loc','original_url','link','route','path')
|
||
|
|
route=_route(original)
|
||
|
|
status=int(row.get('status') or 200)
|
||
|
|
if status not in (200,301,302,307,308,404,410):raise ValueError('不支持的 HTTP 状态')
|
||
|
|
return {'route':route,'original_url':original,'seo_title':_str(row,'seo_title','title','meta_title'),
|
||
|
|
'description':_str(row,'description','seo_description','meta_description'),
|
||
|
|
'canonical':_str(row,'canonical'),'robots':_str(row,'robots'),
|
||
|
|
'h1':_str(row,'h1'),'status':status}
|
||
|
|
|
||
|
|
def _wp_item(element):
|
||
|
|
def get(local,default=''):
|
||
|
|
for ch in element:
|
||
|
|
if ch.tag.rsplit('}',1)[-1]==local:return (ch.text or default)
|
||
|
|
return default
|
||
|
|
def get_content(local):
|
||
|
|
for ch in element:
|
||
|
|
if ch.tag.rsplit('}',1)[-1]==local:return ch.text or ''
|
||
|
|
return ''
|
||
|
|
meta={}
|
||
|
|
for child in element:
|
||
|
|
if child.tag.rsplit('}',1)[-1]=='postmeta':
|
||
|
|
key=value=''
|
||
|
|
for part in child:
|
||
|
|
if part.tag.endswith('meta_key'):key=part.text or ''
|
||
|
|
elif part.tag.endswith('meta_value'):value=part.text or ''
|
||
|
|
if key:meta[key]=value
|
||
|
|
return (get('post_type','post'),get('status','publish'),get('link'),get('title'),get_content('encoded'),meta)
|
||
|
|
|
||
|
|
def from_file(filename:str,blob:bytes):
|
||
|
|
if len(blob)>16_000_000:raise ValueError('源站清单不能超过 16 MB')
|
||
|
|
name=filename.lower();rows=[];assets=[];warnings=[]
|
||
|
|
if name.endswith('.csv'):
|
||
|
|
rows=list(csv.DictReader(io.StringIO(blob.decode('utf-8-sig'))))
|
||
|
|
elif name.endswith('.json'):
|
||
|
|
data=json.loads(blob.decode('utf-8-sig'))
|
||
|
|
rows=data.get('urls',data.get('pages',[])) if isinstance(data,dict) else data
|
||
|
|
if not isinstance(rows,list):raise ValueError('JSON 应包含 pages/urls 数组')
|
||
|
|
elif name.endswith('.xml'):
|
||
|
|
root=ET.fromstring(blob)
|
||
|
|
root_name=root.tag.rsplit('}',1)[-1]
|
||
|
|
if root_name=='urlset':
|
||
|
|
rows=[{'url':next((c.text or '' for c in node if c.tag.rsplit('}',1)[-1]=='loc'),'')} for node in root]
|
||
|
|
elif root_name=='rss':
|
||
|
|
for item in root.findall('./channel/item'):
|
||
|
|
typ,status,url,title,body,meta=_wp_item(item)
|
||
|
|
if typ=='attachment':
|
||
|
|
if url:assets.append((url,''))
|
||
|
|
continue
|
||
|
|
if status!='publish' or typ in ('revision','nav_menu_item'):continue
|
||
|
|
if not url:
|
||
|
|
warnings.append('WXR 某条记录缺少真实链接，无法推断原路径：'+title[:70]);continue
|
||
|
|
rows.append({'url':url,'title':meta.get('_yoast_wpseo_title') or meta.get('rank_math_title') or title,
|
||
|
|
'description':meta.get('_yoast_wpseo_metadesc') or meta.get('rank_math_description') or '',
|
||
|
|
'canonical':meta.get('_yoast_wpseo_canonical') or meta.get('rank_math_canonical_url') or '',
|
||
|
|
'robots':'noindex,follow' if meta.get('_yoast_wpseo_meta-robots-noindex')=='1' else '',
|
||
|
|
'h1':title})
|
||
|
|
soup=BeautifulSoup(body,'html.parser')
|
||
|
|
for img in soup.select('img[src]'):
|
||
|
|
assets.append((img.get('src',''),url))
|
||
|
|
else:raise ValueError('XML 需要 WordPress WXR 或 sitemap.xml')
|
||
|
|
else:raise ValueError('仅支持 .csv / .json / WordPress WXR.xml / sitemap.xml')
|
||
|
|
if not 1<=len(rows)<=MAX_BASELINE:raise ValueError('URL 清单需要1至20000条')
|
||
|
|
out=[];errors=[];seen=set()
|
||
|
|
for i,row in enumerate(rows,1):
|
||
|
|
try:
|
||
|
|
if not isinstance(row,dict):raise ValueError('记录不是对象')
|
||
|
|
normalized=_normalize(row)
|
||
|
|
if normalized['route'] in seen:raise ValueError('原路径重复')
|
||
|
|
seen.add(normalized['route']);out.append(normalized)
|
||
|
|
except Exception as exc:errors.append({'row':i,'message':str(exc)[:180]})
|
||
|
|
return {'records':out,'assets':assets,'errors':errors,'warnings':warnings,'total':len(rows)}
|
||
|
|
|
||
|
|
def save(site_id,manifest,replace=True):
|
||
|
|
if manifest['errors']:raise ValueError('URL 清单存在格式错误，请修正后再导入')
|
||
|
|
with db.connect() as c:
|
||
|
|
if replace:
|
||
|
|
c.execute('DELETE FROM migration_baseline WHERE site_id=?',(site_id,))
|
||
|
|
c.execute('DELETE FROM source_audit_fields WHERE site_id=?',(site_id,))
|
||
|
|
c.execute('DELETE FROM source_audit_assets WHERE site_id=?',(site_id,))
|
||
|
|
for row in manifest['records']:
|
||
|
|
c.execute('''INSERT INTO migration_baseline(site_id,route,seo_title,description,canonical,status)
|
||
|
|
VALUES(?,?,?,?,?,?) ON CONFLICT(site_id,route) DO UPDATE SET
|
||
|
|
seo_title=excluded.seo_title,description=excluded.description,canonical=excluded.canonical,status=excluded.status''',
|
||
|
|
(site_id,row['route'],row['seo_title'],row['description'],row['canonical'],row['status']))
|
||
|
|
c.execute('''INSERT INTO source_audit_fields(site_id,route,robots,h1,original_url) VALUES(?,?,?,?,?)
|
||
|
|
ON CONFLICT(site_id,route) DO UPDATE SET robots=excluded.robots,h1=excluded.h1,original_url=excluded.original_url''',
|
||
|
|
(site_id,row['route'],row['robots'],row['h1'],row['original_url']))
|
||
|
|
for url,origin in manifest['assets'][:MAX_BASELINE]:
|
||
|
|
if not url or len(url)>1000:continue
|
||
|
|
c.execute('INSERT OR IGNORE INTO source_audit_assets VALUES(?,?,?)',(site_id,url,origin[:500]))
|
||
|
|
return {'imported':len(manifest['records']),'asset_references':len(manifest['assets']),'warnings':manifest['warnings']}
|
||
|
|
|
||
|
|
def _tag(soup,name,attrs=None):
|
||
|
|
node=soup.find(name,attrs=attrs or {})
|
||
|
|
return node.get('content','').strip() if node and name=='meta' else node.get_text(' ',strip=True) if node else ''
|
||
|
|
|
||
|
|
def audit(site_id,build_id):
|
||
|
|
# Actual generated bytes, not source entry metadata, are the authority.
|
||
|
|
root=(db.DATA/'builds'/site_id/build_id/'dist').resolve()
|
||
|
|
if not root.is_dir() or not db.one('SELECT id FROM builds WHERE id=? AND site_id=?',(build_id,site_id)):
|
||
|
|
raise ValueError('构建版本不存在')
|
||
|
|
baseline=db.rows('''SELECT b.*,x.robots,x.h1,x.original_url FROM migration_baseline b
|
||
|
|
LEFT JOIN source_audit_fields x ON b.site_id=x.site_id AND b.route=x.route
|
||
|
|
WHERE b.site_id=? ORDER BY b.route''',(site_id,))
|
||
|
|
if not baseline:return {'baseline_count':0,'errors':1,'warnings':0,'passed':False,
|
||
|
|
'issues':[{'severity':'error','field':'baseline','message':'未导入源站 URL 清单'}]}
|
||
|
|
redirects={r['old_path']:(r['new_path'],r['status_code']) for r in db.rows('SELECT old_path,new_path,status_code FROM redirects WHERE site_id=?',(site_id,))}
|
||
|
|
issues=[];matched=0
|
||
|
|
for old in baseline:
|
||
|
|
route=old['route'];expect_status=old['status']; target=builder.output_path(root,route)
|
||
|
|
if expect_status in (301,302,307,308,404,410):
|
||
|
|
if expect_status in (301,302,307,308):
|
||
|
|
if route not in redirects:
|
||
|
|
issues.append({'route':route,'field':'redirect','severity':'error','message':'历史重定向未配置'})
|
||
|
|
elif redirects[route][1]!=expect_status:
|
||
|
|
issues.append({'route':route,'field':'redirect_status','severity':'error','message':'重定向状态码与旧站不一致'})
|
||
|
|
elif target.is_file():
|
||
|
|
issues.append({'route':route,'field':'http_status','severity':'error','message':'旧站的不存在/已删除页面被错误地生成了内容'})
|
||
|
|
continue
|
||
|
|
if not target.is_file():
|
||
|
|
issues.append({'route':route,'field':'url','severity':'error','message':'旧站正常页面在生成网站中缺失'})
|
||
|
|
continue
|
||
|
|
matched+=1
|
||
|
|
soup=BeautifulSoup(target.read_text(encoding='utf-8',errors='replace'),'html.parser')
|
||
|
|
title=_tag(soup,'title')
|
||
|
|
desc_node=soup.find('meta',attrs={'name':re.compile('^description$',re.I)})
|
||
|
|
description=(desc_node.get('content','') if desc_node else '').strip()
|
||
|
|
canonical_node=soup.find('link',rel=lambda value:value and 'canonical' in value)
|
||
|
|
canonical=(canonical_node.get('href','') if canonical_node else '').strip()
|
||
|
|
robot_node=soup.find('meta',attrs={'name':re.compile('^robots$',re.I)})
|
||
|
|
robots=(robot_node.get('content','') if robot_node else '').strip().lower()
|
||
|
|
h1=_tag(soup,'h1')
|
||
|
|
for source,current,label in [('seo_title',title,'title'),('description',description,'description'),('canonical',canonical,'canonical'),('robots',robots,'robots'),('h1',h1,'h1')]:
|
||
|
|
if old.get(source) and ' '.join(str(old[source]).split())!=' '.join(str(current).split()):
|
||
|
|
issues.append({'route':route,'field':label,'severity':'warning','message':'旧新页面字段不一致',
|
||
|
|
'before':str(old[source])[:160],'after':str(current)[:160]})
|
||
|
|
other=__import__('app.quality',fromlist=['report']).report(site_id,build_id)
|
||
|
|
for q in other['issues']:
|
||
|
|
if q['severity']=='error':issues.append({'route':q['file'],'field':q['type'],'severity':'error','message':str(q.get('target','构建质量问题'))[:180]})
|
||
|
|
decisions={x['issue_id']:x for x in db.rows('SELECT * FROM source_audit_exceptions WHERE site_id=? AND build_id=?',(site_id,build_id))}
|
||
|
|
for issue in issues:
|
||
|
|
token=json.dumps([issue.get(k,'') for k in ('route','field','message','before','after')],ensure_ascii=False,sort_keys=True)
|
||
|
|
issue['issue_id']=hashlib.sha256(token.encode()).hexdigest()[:24]
|
||
|
|
approved=decisions.get(issue['issue_id'])
|
||
|
|
issue['reviewed']=bool(approved) if issue['severity']=='warning' else False
|
||
|
|
if approved:
|
||
|
|
issue['reviewer']=approved['reviewer']
|
||
|
|
issue['review_reason']=approved['explanation']
|
||
|
|
outstanding=sum(i['severity']=='warning' and not i['reviewed'] for i in issues)
|
||
|
|
return {'baseline_count':len(baseline),'matched_pages':matched,
|
||
|
|
'errors':sum(x['severity']=='error' for x in issues),
|
||
|
|
'warnings':sum(x['severity']=='warning' for x in issues),
|
||
|
|
'passed':not any(x['severity']=='error' for x in issues),'unreviewed_warnings':outstanding,
|
||
|
|
'quality':{'files':other['files'],'errors':other['errors'],'warnings':other['warnings']},
|
||
|
|
'issues':issues[:2000]}
|
||
|
|
|
||
|
|
@router.post('/sites/{site_id}/import-baseline')
|
||
|
|
async def import_baseline(request:Request,site_id:str,file:UploadFile=File(...)):
|
||
|
|
enterprise.require(request,site_id,'editor')
|
||
|
|
data=await file.read(16_000_001)
|
||
|
|
try:
|
||
|
|
manifest=from_file(file.filename or '',data)
|
||
|
|
if manifest['errors']:
|
||
|
|
return {'imported':0,'errors':manifest['errors'][:200],'total':manifest['total'],'warnings':manifest['warnings']}
|
||
|
|
return save(site_id,manifest)
|
||
|
|
except (ValueError,UnicodeDecodeError) as exc:raise HTTPException(400,str(exc))
|
||
|
|
|
||
|
|
@router.get('/sites/{site_id}/builds/{build_id}/report')
|
||
|
|
def report(request:Request,site_id:str,build_id:str):
|
||
|
|
enterprise.require(request,site_id,'viewer')
|
||
|
|
try:return audit(site_id,build_id)
|
||
|
|
except ValueError as exc:raise HTTPException(404,str(exc))
|
||
|
|
|
||
|
|
@router.post('/sites/{site_id}/builds/{build_id}/review-warning')
|
||
|
|
def approve_warning(request:Request,site_id:str,build_id:str,payload:dict=Body(...)):
|
||
|
|
enterprise.require(request,site_id,'reviewer')
|
||
|
|
issue_id=str(payload.get('issue_id',''))
|
||
|
|
reason=str(payload.get('reason','')).strip()
|
||
|
|
if len(reason)<10 or len(reason)>2000:raise HTTPException(400,'请填写至少10个字符的复核原因及处理依据')
|
||
|
|
evidence=audit(site_id,build_id)
|
||
|
|
issue=next((x for x in evidence['issues'] if x['issue_id']==issue_id and x['severity']=='warning'),None)
|
||
|
|
if not issue:raise HTTPException(404,'只能复核此构建版本当前存在的警告')
|
||
|
|
db.insert('INSERT INTO source_audit_exceptions(site_id,build_id,issue_id,reviewer,explanation,decided_at) VALUES(?,?,?,?,?,?) ON CONFLICT(site_id,build_id,issue_id) DO UPDATE SET reviewer=excluded.reviewer,explanation=excluded.explanation,decided_at=excluded.decided_at',
|
||
|
|
(site_id,build_id,issue_id,request.state.actor,reason,db.now()))
|
||
|
|
return {'approved':True,'issue_id':issue_id,'reviewer':request.state.actor}
|
||
|
|
|
||
|
|
@router.get('/sites/{site_id}/builds/{build_id}/report.csv')
|
||
|
|
def report_csv(request:Request,site_id:str,build_id:str):
|
||
|
|
from fastapi.responses import Response
|
||
|
|
enterprise.require(request,site_id,'viewer')
|
||
|
|
result=audit(site_id,build_id)
|
||
|
|
dest=io.StringIO();writer=csv.DictWriter(dest,fieldnames=['severity','route','field','message','before','after'],extrasaction='ignore')
|
||
|
|
writer.writeheader();writer.writerows(result['issues'])
|
||
|
|
return Response('\ufeff'+dest.getvalue(),media_type='text/csv',headers={'Content-Disposition':'attachment; filename="migration-audit.csv"'})
|