324 lines
17 KiB
Python
324 lines
17 KiB
Python
|
"""SiteForge V8.2: bounded WordPress sitemap discovery, safe media localization
|
|||
|
|
and durable worker-backed REST scanning. All flows require explicit authorization
|
|||
|
|
and remain subject to the existing SiteForge release-review gates.
|
|||
|
|
"""
|
|||
|
|
from __future__ import annotations
|
|||
|
|
import hashlib
|
|||
|
|
import json
|
|||
|
|
import uuid
|
|||
|
|
import zlib
|
|||
|
|
import time
|
|||
|
|
from urllib.parse import urljoin, urlsplit
|
|||
|
|
from bs4 import BeautifulSoup
|
|||
|
|
from defusedxml import ElementTree
|
|||
|
|
from fastapi import APIRouter, Body, HTTPException, Request
|
|||
|
|
from . import db, enterprise, jobs, safe_http, wp_migration_v81 as wp
|
|||
|
|
from .core import route_path
|
|||
|
|
|
|||
|
|
router=APIRouter(prefix='/api/migrations/sites',tags=['V8.2 WordPress migration'])
|
|||
|
|
MAX_SITEMAP_URLS=20_000
|
|||
|
|
MAX_SITEMAP_FILES=60
|
|||
|
|
MAX_MEDIA_BYTES=8_000_000
|
|||
|
|
ALLOWED={'image/png':'.png','image/jpeg':'.jpg','image/webp':'.webp','image/gif':'.gif','image/avif':'.avif'}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def setup():
|
|||
|
|
with db.connect() as c:
|
|||
|
|
c.executescript('''
|
|||
|
|
CREATE TABLE IF NOT EXISTS wp_migration_sitemap(
|
|||
|
|
session_id TEXT NOT NULL REFERENCES wp_migration_sessions(id) ON DELETE CASCADE,
|
|||
|
|
route TEXT NOT NULL, original_url TEXT NOT NULL,
|
|||
|
|
PRIMARY KEY(session_id,route));
|
|||
|
|
CREATE TABLE IF NOT EXISTS wp_migration_media(
|
|||
|
|
session_id TEXT NOT NULL REFERENCES wp_migration_sessions(id) ON DELETE CASCADE,
|
|||
|
|
original_url TEXT NOT NULL, local_url TEXT NOT NULL DEFAULT '',
|
|||
|
|
state TEXT NOT NULL, bytes INTEGER NOT NULL DEFAULT 0,
|
|||
|
|
error TEXT NOT NULL DEFAULT '', updated_at TEXT NOT NULL,
|
|||
|
|
PRIMARY KEY(session_id,original_url));
|
|||
|
|
''')
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _site_url(origin,url):
|
|||
|
|
"""Same host and scoped WordPress base directory (no external sitemap)."""
|
|||
|
|
p=urlsplit(url)
|
|||
|
|
root=urlsplit(origin)
|
|||
|
|
if (p.scheme!=root.scheme or p.username or p.password or p.query or p.fragment
|
|||
|
|
or p.hostname != root.hostname or p.port != root.port):
|
|||
|
|
raise ValueError('仅允许同一 WordPress 站点的无查询参数 Sitemap 地址')
|
|||
|
|
prefix=root.path.rstrip('/')+'/'
|
|||
|
|
if not p.path.startswith(prefix):raise ValueError('Sitemap 超出了来源站点的路径范围')
|
|||
|
|
return url
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _xml_bytes(response):
|
|||
|
|
raw=response.content
|
|||
|
|
if len(raw)>2_000_000:raise ValueError('Sitemap 文件大于 2MB')
|
|||
|
|
if raw[:2]==b'\x1f\x8b':
|
|||
|
|
z=zlib.decompressobj(16+zlib.MAX_WBITS)
|
|||
|
|
raw=z.decompress(raw,3_000_001)
|
|||
|
|
if len(raw)>3_000_000 or z.unconsumed_tail or not z.eof:
|
|||
|
|
raise ValueError('Sitemap 压缩包解压后超出限制')
|
|||
|
|
if len(raw)>3_000_000:raise ValueError('Sitemap 解压后超出限制')
|
|||
|
|
if b'<!DOCTYPE' in raw.upper() or b'<!ENTITY' in raw.upper():
|
|||
|
|
raise ValueError('Sitemap 禁止 DTD/实体声明')
|
|||
|
|
return raw
|
|||
|
|
|
|||
|
|
|
|||
|
|
def discover(site_id,session_id,sitemap_url=None):
|
|||
|
|
s=wp._one(site_id,session_id)
|
|||
|
|
if s['state'] in ('cancelled','completed','completed_with_warnings','running'):
|
|||
|
|
raise HTTPException(409,'当前迁移状态不能修改 Sitemap 清单')
|
|||
|
|
base=s['origin']
|
|||
|
|
start=_site_url(base,sitemap_url or urljoin(base,'wp-sitemap.xml'))
|
|||
|
|
queue=[start];processed=set();urls={}
|
|||
|
|
while queue:
|
|||
|
|
if len(processed)>=MAX_SITEMAP_FILES:raise ValueError('Sitemap 文件数超过上限')
|
|||
|
|
target=queue.pop(0)
|
|||
|
|
if target in processed:continue
|
|||
|
|
_site_url(base,target)
|
|||
|
|
processed.add(target)
|
|||
|
|
response=safe_http.fetch(target,max_bytes=2_000_000,timeout=15,redirects=0,
|
|||
|
|
headers={'Accept':'application/xml,text/xml'})
|
|||
|
|
response.raise_for_status()
|
|||
|
|
root=ElementTree.fromstring(_xml_bytes(response))
|
|||
|
|
root_name=root.tag.split('}')[-1]
|
|||
|
|
if root_name not in ('urlset','sitemapindex'):
|
|||
|
|
raise ValueError('不是有效的 XML Sitemap 或 Sitemap Index')
|
|||
|
|
for node in root:
|
|||
|
|
if node.tag.split('}')[-1] not in ('sitemap','url'):continue
|
|||
|
|
raw=next((v.text.strip() for v in node if v.tag.split('}')[-1]=='loc' and v.text),None)
|
|||
|
|
if not raw:continue
|
|||
|
|
if root_name=='sitemapindex':
|
|||
|
|
queue.append(_site_url(base,raw))
|
|||
|
|
if len(queue)+len(processed)>MAX_SITEMAP_FILES:raise ValueError('Sitemap Index 数量超过上限')
|
|||
|
|
else:
|
|||
|
|
# Different domains and out-of-scope paths cannot enter our baseline.
|
|||
|
|
_site_url(base,raw)
|
|||
|
|
urlp=urlsplit(raw)
|
|||
|
|
normalized=route_path(urlp.path)
|
|||
|
|
if normalized in urls and urls[normalized]!=raw:raise ValueError('Sitemap 中存在冲突 URL: '+normalized)
|
|||
|
|
urls[normalized]=raw
|
|||
|
|
if len(urls)>MAX_SITEMAP_URLS:raise ValueError('Sitemap URL 数量超过上限')
|
|||
|
|
if not urls:raise ValueError('Sitemap 没有找到任何合法页面 URL')
|
|||
|
|
with db.connect() as c:
|
|||
|
|
c.execute('BEGIN IMMEDIATE')
|
|||
|
|
c.execute('DELETE FROM wp_migration_sitemap WHERE session_id=?',(session_id,))
|
|||
|
|
c.executemany('INSERT INTO wp_migration_sitemap(session_id,route,original_url) VALUES(?,?,?)',
|
|||
|
|
((session_id,r,u) for r,u in urls.items()))
|
|||
|
|
return sitemap_status(site_id,session_id)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def sitemap_status(site_id,session_id):
|
|||
|
|
wp._one(site_id,session_id)
|
|||
|
|
total=db.one('SELECT COUNT(*) n FROM wp_migration_sitemap WHERE session_id=?',(session_id,))['n']
|
|||
|
|
missing=db.rows('''SELECT m.route,m.original_url FROM wp_migration_sitemap m
|
|||
|
|
LEFT JOIN wp_migration_urls u ON u.session_id=m.session_id AND u.route=m.route
|
|||
|
|
WHERE m.session_id=? AND u.route IS NULL ORDER BY m.route LIMIT 100''',(session_id,))
|
|||
|
|
missing_count=db.one('''SELECT COUNT(*) n FROM wp_migration_sitemap m LEFT JOIN wp_migration_urls u
|
|||
|
|
ON u.session_id=m.session_id AND u.route=m.route
|
|||
|
|
WHERE m.session_id=? AND u.route IS NULL''',(session_id,))['n']
|
|||
|
|
return {'discovered':total,'missing_count':missing_count,'missing_sample':missing}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _image_valid(mime,raw):
|
|||
|
|
if mime=='image/jpeg':return raw.startswith(b'\xff\xd8\xff')
|
|||
|
|
if mime=='image/png':return raw.startswith(b'\x89PNG\r\n\x1a\n')
|
|||
|
|
if mime=='image/gif':return raw.startswith((b'GIF87a',b'GIF89a'))
|
|||
|
|
if mime=='image/webp':return len(raw)>12 and raw[:4]==b'RIFF' and raw[8:12]==b'WEBP'
|
|||
|
|
if mime=='image/avif':return len(raw)>12 and raw[4:8]==b'ftyp' and b'avif' in raw[8:32]
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _download_image(origin,url,site_id,session_id):
|
|||
|
|
p=urlsplit(url);base=urlsplit(origin)
|
|||
|
|
# Explicitly don't follow offsite CDNs or credential-bearing URLs automatically.
|
|||
|
|
if p.scheme not in ('http','https') or p.hostname!=base.hostname or p.port!=base.port or p.username or p.password:
|
|||
|
|
raise ValueError('外部 CDN 或跨域资源需要单独授权')
|
|||
|
|
r=safe_http.fetch(url,max_bytes=MAX_MEDIA_BYTES,timeout=15,redirects=0)
|
|||
|
|
r.raise_for_status()
|
|||
|
|
mime=r.headers.get('content-type','').split(';',1)[0].lower().strip()
|
|||
|
|
if mime not in ALLOWED or not _image_valid(mime,r.content):raise ValueError('不支持或无效的图片类型')
|
|||
|
|
# Store only simple flat images; never use remote paths as local filenames.
|
|||
|
|
name='wp-'+hashlib.sha256(url.encode()).hexdigest()[:24]+ALLOWED[mime]
|
|||
|
|
target=db.DATA/'media'/site_id/name
|
|||
|
|
target.parent.mkdir(parents=True,exist_ok=True)
|
|||
|
|
if not target.exists():
|
|||
|
|
temp=target.with_suffix(target.suffix+'.'+uuid.uuid4().hex[:8]+'.tmp')
|
|||
|
|
try:temp.write_bytes(r.content);temp.replace(target)
|
|||
|
|
finally:temp.unlink(missing_ok=True)
|
|||
|
|
return '/media/'+name,len(r.content)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _srcset_urls(value):
|
|||
|
|
"""Extract standard WordPress srcset URL candidates, preserving width/density hints."""
|
|||
|
|
if not isinstance(value,str):return []
|
|||
|
|
result=[]
|
|||
|
|
for candidate in value.split(','):
|
|||
|
|
fields=candidate.strip().split()
|
|||
|
|
if fields and fields[0].startswith(('http://','https://','//')):
|
|||
|
|
result.append(fields[0])
|
|||
|
|
return result
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _media_urls(content):
|
|||
|
|
"""Collect all remote media references, including responsive and lazy images."""
|
|||
|
|
urls=[content.get('cover') or '']
|
|||
|
|
soup=BeautifulSoup(content.get('body') or '','html.parser')
|
|||
|
|
for tag in soup.find_all(['img','source','video']):
|
|||
|
|
for key in ('src','data-src','poster'):
|
|||
|
|
if tag.get(key):urls.append(tag[key])
|
|||
|
|
for key in ('srcset','data-srcset'):
|
|||
|
|
urls.extend(_srcset_urls(tag.get(key)))
|
|||
|
|
return list(dict.fromkeys(u for u in urls if isinstance(u,str)
|
|||
|
|
and u.startswith(('https://','http://','//'))))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _rewrite_srcset(value,converted):
|
|||
|
|
candidates=[];changed=False
|
|||
|
|
for part in str(value).split(','):
|
|||
|
|
fields=part.strip().split()
|
|||
|
|
if not fields:continue
|
|||
|
|
url=fields[0]
|
|||
|
|
if url in converted:
|
|||
|
|
fields[0]=converted[url];changed=True
|
|||
|
|
candidates.append(' '.join(fields))
|
|||
|
|
return ', '.join(candidates),changed
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _rewrite_media_html(body,converted):
|
|||
|
|
soup=BeautifulSoup(body,'html.parser');changed=False
|
|||
|
|
for tag in soup.find_all(['img','source','video']):
|
|||
|
|
for key in ('src','data-src','poster'):
|
|||
|
|
if tag.get(key) in converted:
|
|||
|
|
tag[key]=converted[tag[key]];changed=True
|
|||
|
|
for key in ('srcset','data-srcset'):
|
|||
|
|
if not tag.get(key):continue
|
|||
|
|
new,updated=_rewrite_srcset(tag[key],converted)
|
|||
|
|
if updated:
|
|||
|
|
tag[key]=new;changed=True
|
|||
|
|
# A single redundant srcset has no value and can be removed.
|
|||
|
|
if tag.name=='img' and key=='srcset' and len(new.split(','))==1:
|
|||
|
|
fields=new.strip().split()
|
|||
|
|
if fields and fields[0]==tag.get('src'):
|
|||
|
|
tag.attrs.pop(key,None)
|
|||
|
|
return str(soup) if changed else body,changed
|
|||
|
|
|
|||
|
|
|
|||
|
|
def localize_media(site_id,session_id,limit=20):
|
|||
|
|
s=wp._one(site_id,session_id)
|
|||
|
|
if not 1<=int(limit)<=40:raise ValueError('单次处理媒体数量须为 1–40')
|
|||
|
|
if s['state'] in ('running','cancelled','completed','completed_with_warnings'):
|
|||
|
|
raise HTTPException(409,'请先暂停采集，且必须在确认导入前执行媒体本地化')
|
|||
|
|
batches=db.rows('''SELECT b.id FROM wp_migration_pages p JOIN v8_import_batches b ON b.id=p.batch_id
|
|||
|
|
WHERE p.session_id=? AND b.site_id=? AND b.state='staged' ORDER BY p.phase,p.page''',(session_id,site_id))
|
|||
|
|
items=[]
|
|||
|
|
for b in batches:
|
|||
|
|
items.extend(db.rows('SELECT id,content_json FROM v8_import_items WHERE batch_id=? ORDER BY row_number',(b['id'],)))
|
|||
|
|
unique=list(dict.fromkeys(url for item in items
|
|||
|
|
for url in _media_urls(json.loads(item['content_json']))))
|
|||
|
|
if len(unique)>4000:raise ValueError('媒体引用数量超过单次迁移安全上限 4000')
|
|||
|
|
failures=[];done=0
|
|||
|
|
for url in unique:
|
|||
|
|
existing=db.one('SELECT * FROM wp_migration_media WHERE session_id=? AND original_url=?',(session_id,url))
|
|||
|
|
if existing and existing['state']=='localized':continue
|
|||
|
|
if done>=int(limit):break
|
|||
|
|
done+=1
|
|||
|
|
try:
|
|||
|
|
local,size=_download_image(s['origin'],url,site_id,session_id)
|
|||
|
|
state='localized';error=''
|
|||
|
|
except Exception as exc:
|
|||
|
|
local='';size=0;state='review_required';error=str(exc)[:240]
|
|||
|
|
failures.append({'url':url,'reason':error})
|
|||
|
|
db.execute('''INSERT INTO wp_migration_media(session_id,original_url,local_url,state,bytes,error,updated_at)
|
|||
|
|
VALUES(?,?,?,?,?,?,?) ON CONFLICT(session_id,original_url) DO UPDATE SET
|
|||
|
|
local_url=excluded.local_url,state=excluded.state,bytes=excluded.bytes,
|
|||
|
|
error=excluded.error,updated_at=excluded.updated_at''',
|
|||
|
|
(session_id,url,local,state,size,error,db.now()))
|
|||
|
|
# Rewrite staged content only, never an already-committed or hand-edited entry.
|
|||
|
|
converted={r['original_url']:r['local_url'] for r in db.rows('SELECT * FROM wp_migration_media WHERE session_id=? AND state=?',(session_id,'localized'))}
|
|||
|
|
with db.connect() as c:
|
|||
|
|
c.execute('BEGIN IMMEDIATE')
|
|||
|
|
for item in items:
|
|||
|
|
obj=json.loads(item['content_json']);modified=False
|
|||
|
|
if obj.get('cover') in converted:obj['cover']=converted[obj['cover']];modified=True
|
|||
|
|
if obj.get('body'):
|
|||
|
|
updated,replaced=_rewrite_media_html(obj['body'],converted)
|
|||
|
|
if replaced:obj['body']=updated;modified=True
|
|||
|
|
if modified:c.execute('UPDATE v8_import_items SET content_json=? WHERE id=?',(json.dumps(obj,ensure_ascii=False),item['id']))
|
|||
|
|
return {'attempted':done,'localized':len(converted),'failed':failures,'total_candidates':len(unique),
|
|||
|
|
'remaining':len([x for x in unique if x not in converted]),'requires_review':bool(failures)}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _findings(site_id,session_id):
|
|||
|
|
# Sitemap-only URLs must block claims of complete migration.
|
|||
|
|
sitemap=sitemap_status(site_id,session_id)
|
|||
|
|
issues=[f'站点地图中未由 REST 采集的页面：{r["route"]}' for r in sitemap['missing_sample']]
|
|||
|
|
if sitemap['missing_count']>len(sitemap['missing_sample']):
|
|||
|
|
issues.append(f'其他未采集 Sitemap URL 数量：{sitemap["missing_count"]-len(sitemap["missing_sample"])}')
|
|||
|
|
for r in db.rows('SELECT original_url,error FROM wp_migration_media WHERE session_id=? AND state!=?',
|
|||
|
|
(session_id,'localized')):
|
|||
|
|
issues.append('媒体未本地化：'+r['original_url'][:170]+'；'+r['error'][:90])
|
|||
|
|
for b in db.rows('''SELECT p.batch_id FROM wp_migration_pages p JOIN v8_import_batches b ON b.id=p.batch_id
|
|||
|
|
WHERE p.session_id=? AND b.site_id=?''',(session_id,site_id)):
|
|||
|
|
if not b['batch_id']:continue
|
|||
|
|
for record in db.rows('SELECT route,content_json FROM v8_import_items WHERE batch_id=?',(b['batch_id'],)):
|
|||
|
|
content=json.loads(record['content_json'])
|
|||
|
|
urls=_media_urls(content)
|
|||
|
|
if urls:
|
|||
|
|
issues.append('正文或封面仍引用外部图片：'+record['route'])
|
|||
|
|
return list(dict.fromkeys(issues))[:500]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def submit_scan(site_id,session_id,actor,key=None):
|
|||
|
|
s=wp._one(site_id,session_id)
|
|||
|
|
if s['state'] in ('ready','cancelled','completed','completed_with_warnings','limit_reached'):
|
|||
|
|
raise HTTPException(409,'迁移采集已经完成或被取消')
|
|||
|
|
return jobs.submit_migration(site_id,session_id,actor,key)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def run_scan(site_id,session_id,job_id,max_pages=500):
|
|||
|
|
"""Run by isolated Worker subprocess only; do not auto-commit staged data."""
|
|||
|
|
count=0
|
|||
|
|
while count<max_pages:
|
|||
|
|
job=db.one('SELECT cancel_requested FROM build_jobs WHERE id=? AND site_id=?',(job_id,site_id))
|
|||
|
|
if job is None or job['cancel_requested']:return {'state':'cancelled','pages_processed':count}
|
|||
|
|
s=wp._one(site_id,session_id)
|
|||
|
|
if s['state']=='running' and s['lease_until'] and s['lease_until']>db.now():
|
|||
|
|
# Previous worker may have died while fetching; wait for its per-page
|
|||
|
|
# lease instead of burning all task retries during the 60-second lease.
|
|||
|
|
waited=0
|
|||
|
|
while waited<70 and wp._one(site_id,session_id)['lease_until'] and wp._one(site_id,session_id)['lease_until']>db.now():
|
|||
|
|
row=db.one('SELECT cancel_requested FROM build_jobs WHERE id=?',(job_id,))
|
|||
|
|
if not row or row['cancel_requested']:return {'state':'cancelled','pages_processed':count}
|
|||
|
|
time.sleep(1);waited+=1
|
|||
|
|
if waited>=70:raise TimeoutError('迁移页锁未释放，等待后再重试')
|
|||
|
|
if s['state'] in ('ready','cancelled','limit_reached','completed','completed_with_warnings'):
|
|||
|
|
return {'state':s['state'],'pages_processed':count,'fetched':s['fetched']}
|
|||
|
|
wp.advance(site_id,session_id)
|
|||
|
|
count+=1
|
|||
|
|
return {'state':'page_limit','pages_processed':count}
|
|||
|
|
|
|||
|
|
|
|||
|
|
@router.post('/{site_id}/wordpress/sessions/{session_id}/sitemap')
|
|||
|
|
def api_sitemap(request:Request,site_id:str,session_id:str,payload:dict=Body(default_factory=dict)):
|
|||
|
|
enterprise.require(request,site_id,'editor')
|
|||
|
|
try:return discover(site_id,session_id,payload.get('url'))
|
|||
|
|
except (ValueError,safe_http.OutboundError,ElementTree.ParseError) as exc:raise HTTPException(400,str(exc)) from exc
|
|||
|
|
|
|||
|
|
@router.get('/{site_id}/wordpress/sessions/{session_id}/sitemap')
|
|||
|
|
def api_sitemap_status(request:Request,site_id:str,session_id:str):
|
|||
|
|
enterprise.require(request,site_id,'viewer')
|
|||
|
|
return sitemap_status(site_id,session_id)
|
|||
|
|
|
|||
|
|
@router.post('/{site_id}/wordpress/sessions/{session_id}/media/localize')
|
|||
|
|
def api_media(request:Request,site_id:str,session_id:str,payload:dict=Body(default_factory=dict)):
|
|||
|
|
enterprise.require(request,site_id,'editor')
|
|||
|
|
try:return localize_media(site_id,session_id,int(payload.get('limit',20)))
|
|||
|
|
except (ValueError,TypeError) as exc:raise HTTPException(400,str(exc)) from exc
|
|||
|
|
|
|||
|
|
@router.post('/{site_id}/wordpress/sessions/{session_id}/scan')
|
|||
|
|
def api_scan(request:Request,site_id:str,session_id:str,payload:dict=Body(default_factory=dict)):
|
|||
|
|
enterprise.require(request,site_id,'editor')
|
|||
|
|
try:return submit_scan(site_id,session_id,request.state.actor,payload.get('idempotency_key'))
|
|||
|
|
except (jobs.JobConflict,jobs.JobCapacityError) as exc:raise HTTPException(409,str(exc)) from exc
|