浅念
601451f22b
Overlay ~150 upstream items onto our V8.12 base, preserving trade-pro
theme, registry tests, and design docs. Re-applied our patch set
(discovery/env_for/blocks.css/type-mapping/theme_color/frontend entries,
test_theme_pro_v3 mods) — all anchors survived upstream evolution.
New platform enhancement (fixes upgrade-introduced regression):
multilingual.site_locale now honors settings_json.locale, plus
PUT /api/i18n/sites/{id}/locale — standalone English sites get
<html lang='en'> under the V8.15 locale-rewrite pipeline.
compose: adopt V8.21.3 (image 8.21.3-dev.1, /api/ready probes, MySQL/S3
env) with project name pinned to siteforge-v8-10 (data volumes).
Tests: 356 passed / 11 failed — all 11 confirmed pre-existing in
pristine V8.21.3 on this host (10 upstream Windows env + our build test
now green after locale fix). Patch registry updated in plan doc.
Co-Authored-By: Claude Code <noreply@anthropic.com>
351 lines
18 KiB
Python
351 lines
18 KiB
Python
"""SiteForge V8.2: bounded WordPress sitemap discovery, safe media localization
|
||
and durable worker-backed REST scanning. All flows require explicit authorization
|
||
and remain subject to the existing SiteForge release-review gates.
|
||
"""
|
||
from __future__ import annotations
|
||
import re
|
||
import hashlib
|
||
import json
|
||
import uuid
|
||
import zlib
|
||
import time
|
||
from urllib.parse import urljoin, urlsplit
|
||
from bs4 import BeautifulSoup
|
||
from defusedxml import ElementTree
|
||
from fastapi import APIRouter, Body, HTTPException, Request
|
||
from . import db, enterprise, jobs, safe_http, wp_migration_v81 as wp
|
||
from .core import route_path
|
||
|
||
router=APIRouter(prefix='/api/migrations/sites',tags=['V8.2 WordPress migration'])
|
||
MAX_SITEMAP_URLS=20_000
|
||
MAX_SITEMAP_FILES=60
|
||
MAX_MEDIA_BYTES=8_000_000
|
||
ALLOWED={'image/png':'.png','image/jpeg':'.jpg','image/webp':'.webp','image/gif':'.gif','image/avif':'.avif'}
|
||
|
||
|
||
def setup():
|
||
with db.connect() as c:
|
||
c.executescript('''
|
||
CREATE TABLE IF NOT EXISTS wp_migration_sitemap(
|
||
session_id TEXT NOT NULL REFERENCES wp_migration_sessions(id) ON DELETE CASCADE,
|
||
route TEXT NOT NULL, original_url TEXT NOT NULL,
|
||
PRIMARY KEY(session_id,route));
|
||
CREATE TABLE IF NOT EXISTS wp_migration_media(
|
||
session_id TEXT NOT NULL REFERENCES wp_migration_sessions(id) ON DELETE CASCADE,
|
||
original_url TEXT NOT NULL, local_url TEXT NOT NULL DEFAULT '',
|
||
state TEXT NOT NULL, bytes INTEGER NOT NULL DEFAULT 0,
|
||
error TEXT NOT NULL DEFAULT '', updated_at TEXT NOT NULL,
|
||
PRIMARY KEY(session_id,original_url));
|
||
''')
|
||
|
||
|
||
def _site_url(origin,url):
|
||
"""Same host and scoped WordPress base directory (no external sitemap)."""
|
||
p=urlsplit(url)
|
||
root=urlsplit(origin)
|
||
if (p.scheme!=root.scheme or p.username or p.password or p.query or p.fragment
|
||
or p.hostname != root.hostname or p.port != root.port):
|
||
raise ValueError('仅允许同一 WordPress 站点的无查询参数 Sitemap 地址')
|
||
prefix=root.path.rstrip('/')+'/'
|
||
if not p.path.startswith(prefix):raise ValueError('Sitemap 超出了来源站点的路径范围')
|
||
return url
|
||
|
||
|
||
def _xml_bytes(response):
|
||
raw=response.content
|
||
if len(raw)>2_000_000:raise ValueError('Sitemap 文件大于 2MB')
|
||
if raw[:2]==b'\x1f\x8b':
|
||
z=zlib.decompressobj(16+zlib.MAX_WBITS)
|
||
raw=z.decompress(raw,3_000_001)
|
||
if len(raw)>3_000_000 or z.unconsumed_tail or not z.eof:
|
||
raise ValueError('Sitemap 压缩包解压后超出限制')
|
||
if len(raw)>3_000_000:raise ValueError('Sitemap 解压后超出限制')
|
||
if b'<!DOCTYPE' in raw.upper() or b'<!ENTITY' in raw.upper():
|
||
raise ValueError('Sitemap 禁止 DTD/实体声明')
|
||
return raw
|
||
|
||
|
||
def discover(site_id,session_id,sitemap_url=None):
|
||
s=wp._one(site_id,session_id)
|
||
if s['state'] in ('cancelled','completed','completed_with_warnings','running'):
|
||
raise HTTPException(409,'当前迁移状态不能修改 Sitemap 清单')
|
||
base=s['origin']
|
||
start=_site_url(base,sitemap_url or urljoin(base,'wp-sitemap.xml'))
|
||
queue=[start];processed=set();urls={}
|
||
while queue:
|
||
if len(processed)>=MAX_SITEMAP_FILES:raise ValueError('Sitemap 文件数超过上限')
|
||
target=queue.pop(0)
|
||
if target in processed:continue
|
||
_site_url(base,target)
|
||
processed.add(target)
|
||
response=safe_http.fetch(target,max_bytes=2_000_000,timeout=15,redirects=0,
|
||
headers={'Accept':'application/xml,text/xml'})
|
||
response.raise_for_status()
|
||
root=ElementTree.fromstring(_xml_bytes(response))
|
||
root_name=root.tag.split('}')[-1]
|
||
if root_name not in ('urlset','sitemapindex'):
|
||
raise ValueError('不是有效的 XML Sitemap 或 Sitemap Index')
|
||
for node in root:
|
||
if node.tag.split('}')[-1] not in ('sitemap','url'):continue
|
||
raw=next((v.text.strip() for v in node if v.tag.split('}')[-1]=='loc' and v.text),None)
|
||
if not raw:continue
|
||
if root_name=='sitemapindex':
|
||
queue.append(_site_url(base,raw))
|
||
if len(queue)+len(processed)>MAX_SITEMAP_FILES:raise ValueError('Sitemap Index 数量超过上限')
|
||
else:
|
||
# Different domains and out-of-scope paths cannot enter our baseline.
|
||
_site_url(base,raw)
|
||
urlp=urlsplit(raw)
|
||
normalized=route_path(urlp.path)
|
||
if normalized in urls and urls[normalized]!=raw:raise ValueError('Sitemap 中存在冲突 URL: '+normalized)
|
||
urls[normalized]=raw
|
||
if len(urls)>MAX_SITEMAP_URLS:raise ValueError('Sitemap URL 数量超过上限')
|
||
if not urls:raise ValueError('Sitemap 没有找到任何合法页面 URL')
|
||
with db.connect() as c:
|
||
c.execute('BEGIN IMMEDIATE')
|
||
c.execute('DELETE FROM wp_migration_sitemap WHERE session_id=?',(session_id,))
|
||
c.executemany('INSERT INTO wp_migration_sitemap(session_id,route,original_url) VALUES(?,?,?)',
|
||
((session_id,r,u) for r,u in urls.items()))
|
||
return sitemap_status(site_id,session_id)
|
||
|
||
|
||
def sitemap_status(site_id,session_id):
|
||
wp._one(site_id,session_id)
|
||
total=db.one('SELECT COUNT(*) n FROM wp_migration_sitemap WHERE session_id=?',(session_id,))['n']
|
||
missing=db.rows('''SELECT m.route,m.original_url FROM wp_migration_sitemap m
|
||
LEFT JOIN wp_migration_urls u ON u.session_id=m.session_id AND u.route=m.route
|
||
WHERE m.session_id=? AND u.route IS NULL ORDER BY m.route LIMIT 100''',(session_id,))
|
||
missing_count=db.one('''SELECT COUNT(*) n FROM wp_migration_sitemap m LEFT JOIN wp_migration_urls u
|
||
ON u.session_id=m.session_id AND u.route=m.route
|
||
WHERE m.session_id=? AND u.route IS NULL''',(session_id,))['n']
|
||
return {'discovered':total,'missing_count':missing_count,'missing_sample':missing}
|
||
|
||
|
||
def _image_valid(mime,raw):
|
||
if mime=='image/jpeg':return raw.startswith(b'\xff\xd8\xff')
|
||
if mime=='image/png':return raw.startswith(b'\x89PNG\r\n\x1a\n')
|
||
if mime=='image/gif':return raw.startswith((b'GIF87a',b'GIF89a'))
|
||
if mime=='image/webp':return len(raw)>12 and raw[:4]==b'RIFF' and raw[8:12]==b'WEBP'
|
||
if mime=='image/avif':return len(raw)>12 and raw[4:8]==b'ftyp' and b'avif' in raw[8:32]
|
||
return False
|
||
|
||
|
||
def _download_image(origin,url,site_id,session_id):
|
||
p=urlsplit(url);base=urlsplit(origin)
|
||
# Explicitly don't follow offsite CDNs or credential-bearing URLs automatically.
|
||
if p.scheme not in ('http','https') or p.hostname!=base.hostname or p.port!=base.port or p.username or p.password:
|
||
raise ValueError('外部 CDN 或跨域资源需要单独授权')
|
||
r=safe_http.fetch(url,max_bytes=MAX_MEDIA_BYTES,timeout=15,redirects=0)
|
||
r.raise_for_status()
|
||
mime=r.headers.get('content-type','').split(';',1)[0].lower().strip()
|
||
if mime not in ALLOWED or not _image_valid(mime,r.content):raise ValueError('不支持或无效的图片类型')
|
||
# Store only simple flat images; never use remote paths as local filenames.
|
||
name='wp-'+hashlib.sha256(url.encode()).hexdigest()[:24]+ALLOWED[mime]
|
||
target=db.DATA/'media'/site_id/name
|
||
target.parent.mkdir(parents=True,exist_ok=True)
|
||
if not target.exists():
|
||
temp=target.with_suffix(target.suffix+'.'+uuid.uuid4().hex[:8]+'.tmp')
|
||
try:temp.write_bytes(r.content);temp.replace(target)
|
||
finally:temp.unlink(missing_ok=True)
|
||
return '/media/'+name,len(r.content)
|
||
|
||
|
||
def _srcset_urls(value):
|
||
"""Extract standard WordPress srcset URL candidates, preserving width/density hints."""
|
||
if not isinstance(value,str):return []
|
||
result=[]
|
||
for candidate in value.split(','):
|
||
fields=candidate.strip().split()
|
||
if fields and fields[0].startswith(('http://','https://','//')):
|
||
result.append(fields[0])
|
||
return result
|
||
|
||
|
||
# CSS background images are common in Elementor containers and ACF galleries.
|
||
# Restrict to syntactically explicit remote URLs, never evaluate arbitrary CSS.
|
||
CSS_URL = re.compile(r'''url\(\s*([\"\']?)((?:https?:)?//[^\"\')\s]+)\1\s*\)''', re.I)
|
||
|
||
|
||
def _css_urls(raw):
|
||
return [match.group(2) for match in CSS_URL.finditer(str(raw or ''))]
|
||
|
||
|
||
def _rewrite_css_urls(raw,converted):
|
||
def replace(match):
|
||
url=match.group(2)
|
||
if url not in converted:return match.group(0)
|
||
quote=match.group(1)
|
||
return f'url({quote}{converted[url]}{quote})'
|
||
updated=CSS_URL.sub(replace,str(raw or ''))
|
||
return updated,updated!=str(raw or '')
|
||
|
||
|
||
def _media_urls(content):
|
||
"""Collect images plus responsive/lazy/CSS backgrounds for explicit review."""
|
||
urls=[content.get('cover') or '']
|
||
soup=BeautifulSoup(content.get('body') or '','html.parser')
|
||
for tag in soup.find_all(True):
|
||
for key in ('src','data-src','poster','data-background','data-background-image'):
|
||
if tag.get(key):urls.append(tag[key])
|
||
for key in ('srcset','data-srcset'):
|
||
urls.extend(_srcset_urls(tag.get(key)))
|
||
if tag.get('style'):urls.extend(_css_urls(tag['style']))
|
||
if tag.name=='style':urls.extend(_css_urls(tag.get_text()))
|
||
return list(dict.fromkeys(u for u in urls if isinstance(u,str)
|
||
and u.startswith(('https://','http://','//'))))
|
||
|
||
|
||
def _rewrite_srcset(value,converted):
|
||
candidates=[];changed=False
|
||
for part in str(value).split(','):
|
||
fields=part.strip().split()
|
||
if not fields:continue
|
||
url=fields[0]
|
||
if url in converted:
|
||
fields[0]=converted[url];changed=True
|
||
candidates.append(' '.join(fields))
|
||
return ', '.join(candidates),changed
|
||
|
||
|
||
def _rewrite_media_html(body,converted):
|
||
soup=BeautifulSoup(body,'html.parser');changed=False
|
||
for tag in soup.find_all(True):
|
||
for key in ('src','data-src','poster','data-background','data-background-image'):
|
||
if tag.get(key) in converted:
|
||
tag[key]=converted[tag[key]];changed=True
|
||
for key in ('srcset','data-srcset'):
|
||
if not tag.get(key):continue
|
||
new,updated=_rewrite_srcset(tag[key],converted)
|
||
if updated:
|
||
tag[key]=new;changed=True
|
||
if tag.name=='img' and key=='srcset' and len(new.split(','))==1:
|
||
fields=new.strip().split()
|
||
if fields and fields[0]==tag.get('src'):
|
||
tag.attrs.pop(key,None)
|
||
if tag.get('style'):
|
||
value,updated=_rewrite_css_urls(tag['style'],converted)
|
||
if updated:tag['style']=value;changed=True
|
||
if tag.name=='style':
|
||
value,updated=_rewrite_css_urls(tag.string if tag.string is not None else tag.get_text(),converted)
|
||
if updated:tag.string=value;changed=True
|
||
return str(soup) if changed else body,changed
|
||
|
||
|
||
def localize_media(site_id,session_id,limit=20):
|
||
s=wp._one(site_id,session_id)
|
||
if not 1<=int(limit)<=40:raise ValueError('单次处理媒体数量须为 1–40')
|
||
if s['state'] in ('running','cancelled','completed','completed_with_warnings'):
|
||
raise HTTPException(409,'请先暂停采集，且必须在确认导入前执行媒体本地化')
|
||
batches=db.rows('''SELECT b.id FROM wp_migration_pages p JOIN v8_import_batches b ON b.id=p.batch_id
|
||
WHERE p.session_id=? AND b.site_id=? AND b.state='staged' ORDER BY p.phase,p.page''',(session_id,site_id))
|
||
items=[]
|
||
for b in batches:
|
||
items.extend(db.rows('SELECT id,content_json FROM v8_import_items WHERE batch_id=? ORDER BY row_number',(b['id'],)))
|
||
unique=list(dict.fromkeys(url for item in items
|
||
for url in _media_urls(json.loads(item['content_json']))))
|
||
if len(unique)>4000:raise ValueError('媒体引用数量超过单次迁移安全上限 4000')
|
||
failures=[];done=0
|
||
for url in unique:
|
||
existing=db.one('SELECT * FROM wp_migration_media WHERE session_id=? AND original_url=?',(session_id,url))
|
||
if existing and existing['state']=='localized':continue
|
||
if done>=int(limit):break
|
||
done+=1
|
||
try:
|
||
local,size=_download_image(s['origin'],url,site_id,session_id)
|
||
state='localized';error=''
|
||
except Exception as exc:
|
||
local='';size=0;state='review_required';error=str(exc)[:240]
|
||
failures.append({'url':url,'reason':error})
|
||
db.execute('''INSERT INTO wp_migration_media(session_id,original_url,local_url,state,bytes,error,updated_at)
|
||
VALUES(?,?,?,?,?,?,?) ON CONFLICT(session_id,original_url) DO UPDATE SET
|
||
local_url=excluded.local_url,state=excluded.state,bytes=excluded.bytes,
|
||
error=excluded.error,updated_at=excluded.updated_at''',
|
||
(session_id,url,local,state,size,error,db.now()))
|
||
# Rewrite staged content only, never an already-committed or hand-edited entry.
|
||
converted={r['original_url']:r['local_url'] for r in db.rows('SELECT * FROM wp_migration_media WHERE session_id=? AND state=?',(session_id,'localized'))}
|
||
with db.connect() as c:
|
||
c.execute('BEGIN IMMEDIATE')
|
||
for item in items:
|
||
obj=json.loads(item['content_json']);modified=False
|
||
if obj.get('cover') in converted:obj['cover']=converted[obj['cover']];modified=True
|
||
if obj.get('body'):
|
||
updated,replaced=_rewrite_media_html(obj['body'],converted)
|
||
if replaced:obj['body']=updated;modified=True
|
||
if modified:c.execute('UPDATE v8_import_items SET content_json=? WHERE id=?',(json.dumps(obj,ensure_ascii=False),item['id']))
|
||
return {'attempted':done,'localized':len(converted),'failed':failures,'total_candidates':len(unique),
|
||
'remaining':len([x for x in unique if x not in converted]),'requires_review':bool(failures)}
|
||
|
||
|
||
def _findings(site_id,session_id):
|
||
# Sitemap-only URLs must block claims of complete migration.
|
||
sitemap=sitemap_status(site_id,session_id)
|
||
issues=[f'站点地图中未由 REST 采集的页面：{r["route"]}' for r in sitemap['missing_sample']]
|
||
if sitemap['missing_count']>len(sitemap['missing_sample']):
|
||
issues.append(f'其他未采集 Sitemap URL 数量：{sitemap["missing_count"]-len(sitemap["missing_sample"])}')
|
||
for r in db.rows('SELECT original_url,error FROM wp_migration_media WHERE session_id=? AND state!=?',
|
||
(session_id,'localized')):
|
||
issues.append('媒体未本地化：'+r['original_url'][:170]+'；'+r['error'][:90])
|
||
for b in db.rows('''SELECT p.batch_id FROM wp_migration_pages p JOIN v8_import_batches b ON b.id=p.batch_id
|
||
WHERE p.session_id=? AND b.site_id=?''',(session_id,site_id)):
|
||
if not b['batch_id']:continue
|
||
for record in db.rows('SELECT route,content_json FROM v8_import_items WHERE batch_id=?',(b['batch_id'],)):
|
||
content=json.loads(record['content_json'])
|
||
urls=_media_urls(content)
|
||
if urls:
|
||
issues.append('正文或封面仍引用外部图片：'+record['route'])
|
||
return list(dict.fromkeys(issues))[:500]
|
||
|
||
|
||
def submit_scan(site_id,session_id,actor,key=None):
|
||
s=wp._one(site_id,session_id)
|
||
if s['state'] in ('ready','cancelled','completed','completed_with_warnings','limit_reached'):
|
||
raise HTTPException(409,'迁移采集已经完成或被取消')
|
||
return jobs.submit_migration(site_id,session_id,actor,key)
|
||
|
||
|
||
def run_scan(site_id,session_id,job_id,max_pages=500):
|
||
"""Run by isolated Worker subprocess only; do not auto-commit staged data."""
|
||
count=0
|
||
while count<max_pages:
|
||
job=db.one('SELECT cancel_requested FROM build_jobs WHERE id=? AND site_id=?',(job_id,site_id))
|
||
if job is None or job['cancel_requested']:return {'state':'cancelled','pages_processed':count}
|
||
s=wp._one(site_id,session_id)
|
||
if s['state']=='running' and s['lease_until'] and s['lease_until']>db.now():
|
||
# Previous worker may have died while fetching; wait for its per-page
|
||
# lease instead of burning all task retries during the 60-second lease.
|
||
waited=0
|
||
while waited<70 and wp._one(site_id,session_id)['lease_until'] and wp._one(site_id,session_id)['lease_until']>db.now():
|
||
row=db.one('SELECT cancel_requested FROM build_jobs WHERE id=?',(job_id,))
|
||
if not row or row['cancel_requested']:return {'state':'cancelled','pages_processed':count}
|
||
time.sleep(1);waited+=1
|
||
if waited>=70:raise TimeoutError('迁移页锁未释放，等待后再重试')
|
||
if s['state'] in ('ready','cancelled','limit_reached','completed','completed_with_warnings'):
|
||
return {'state':s['state'],'pages_processed':count,'fetched':s['fetched']}
|
||
wp.advance(site_id,session_id)
|
||
count+=1
|
||
return {'state':'page_limit','pages_processed':count}
|
||
|
||
|
||
@router.post('/{site_id}/wordpress/sessions/{session_id}/sitemap')
|
||
def api_sitemap(request:Request,site_id:str,session_id:str,payload:dict=Body(default_factory=dict)):
|
||
enterprise.require(request,site_id,'editor')
|
||
try:return discover(site_id,session_id,payload.get('url'))
|
||
except (ValueError,safe_http.OutboundError,ElementTree.ParseError) as exc:raise HTTPException(400,str(exc)) from exc
|
||
|
||
@router.get('/{site_id}/wordpress/sessions/{session_id}/sitemap')
|
||
def api_sitemap_status(request:Request,site_id:str,session_id:str):
|
||
enterprise.require(request,site_id,'viewer')
|
||
return sitemap_status(site_id,session_id)
|
||
|
||
@router.post('/{site_id}/wordpress/sessions/{session_id}/media/localize')
|
||
def api_media(request:Request,site_id:str,session_id:str,payload:dict=Body(default_factory=dict)):
|
||
enterprise.require(request,site_id,'editor')
|
||
try:return localize_media(site_id,session_id,int(payload.get('limit',20)))
|
||
except (ValueError,TypeError) as exc:raise HTTPException(400,str(exc)) from exc
|
||
|
||
@router.post('/{site_id}/wordpress/sessions/{session_id}/scan')
|
||
def api_scan(request:Request,site_id:str,session_id:str,payload:dict=Body(default_factory=dict)):
|
||
enterprise.require(request,site_id,'editor')
|
||
try:return submit_scan(site_id,session_id,request.state.actor,payload.get('idempotency_key'))
|
||
except (jobs.JobConflict,jobs.JobCapacityError) as exc:raise HTTPException(409,str(exc)) from exc
|