浅念
8cea7dee20
- SiteForge Enterprise V8.10.0-dev.1 upstream source (210 tests) - Fix: v8_workflows._mapped_input now reads both 类型/分类 columns so WXR imports keep post/product types instead of collapsing to page - Docs: trade-pro theme port spec (R3) and implementation plan Co-Authored-By: Claude Code <noreply@anthropic.com>
100 lines
5.6 KiB
Python
100 lines
5.6 KiB
Python
"""CMS adapter framework: bounded public-site detection and WP REST import.
|
|
|
|
Private/paid/SSO content is intentionally not accessed by this adapter. Other
|
|
platforms should provide a compatible adapter for their authorised APIs.
|
|
"""
|
|
from dataclasses import dataclass
|
|
from urllib.parse import urljoin,urlparse
|
|
from bs4 import BeautifulSoup
|
|
import httpx
|
|
from .core import ensure_public_url, route_path, save_entry
|
|
from . import db, safe_http
|
|
|
|
USER_AGENT='SiteForge/1.0 (+authorized-migration)'
|
|
|
|
@dataclass(frozen=True)
|
|
class SiteAnalysis:
|
|
url:str
|
|
platform:str
|
|
title:str
|
|
rest_api:str|None
|
|
hints:list[str]
|
|
|
|
def _get(client,url):
|
|
# TCP is pinned to the public IP returned by the security validation,
|
|
# while HTTPS still validates against the original hostname.
|
|
response=safe_http.fetch(url,max_bytes=4_000_000,timeout=13,redirects=0)
|
|
response.raise_for_status()
|
|
return response
|
|
|
|
def analyze(url):
|
|
ensure_public_url(url)
|
|
with httpx.Client(timeout=httpx.Timeout(13,connect=5),trust_env=False,headers={'User-Agent':USER_AGENT}) as client:
|
|
r=_get(client,url)
|
|
soup=BeautifulSoup(r.text,'html.parser')
|
|
generator=soup.find('meta',attrs={'name':'generator'})
|
|
gen=(generator.get('content','') if generator else '').lower()
|
|
rest=soup.find('link',rel='https://api.w.org/')
|
|
api_url=rest.get('href') if rest else None
|
|
platform='WordPress' if 'wordpress' in gen or api_url or 'wp-content/' in r.text[:200000] else ('Drupal' if 'drupal' in gen else 'Joomla' if 'joomla' in gen else 'Unknown / Custom')
|
|
hints=[]
|
|
if soup.find('meta',attrs={'name':'robots'}):hints.append('检测到 robots meta')
|
|
if soup.find('link',rel='canonical'):hints.append('检测到 Canonical')
|
|
if soup.find('script',attrs={'type':'application/ld+json'}):hints.append('检测到结构化数据')
|
|
if not api_url and platform=='WordPress':api_url=urljoin(url,'/wp-json/')
|
|
return SiteAnalysis(url,platform,soup.title.get_text(' ',strip=True)[:200] if soup.title else '',api_url,hints).__dict__
|
|
|
|
def import_wordpress_rest(site_id,origin,max_items=150,on_conflict='skip'):
|
|
"""Import public published posts/pages from a WP REST API, preserving URL paths.
|
|
|
|
Special Elementor/ACF/Yoast fields require additional adapter support.
|
|
"""
|
|
if max_items<1 or max_items>1000:raise ValueError('允许导入 1-1000 条公开内容')
|
|
if on_conflict not in ('skip','update'):raise ValueError('无效的冲突策略')
|
|
root=origin.rstrip('/')+'/'
|
|
ensure_public_url(root)
|
|
result={'imported':0,'updated':0,'skipped':0,'errors':[]}
|
|
with httpx.Client(timeout=httpx.Timeout(18,connect=6),trust_env=False,headers={'User-Agent':USER_AGENT}) as client:
|
|
for kind in ('pages','posts'):
|
|
batch_size=min(100,max_items)
|
|
remaining=max_items-(result['imported']+result['updated']+result['skipped'])
|
|
if remaining<=0:break
|
|
page=1
|
|
while remaining>0 and page<=20:
|
|
api=urljoin(root,f'wp-json/wp/v2/{kind}?per_page={batch_size}&page={page}&_embed=1')
|
|
try:
|
|
response=_get(client,api)
|
|
content=response.json()
|
|
if not isinstance(content,list):raise ValueError('API 返回内容不是数组')
|
|
except safe_http.OutboundError as exc:
|
|
if str(exc) in ('Upstream HTTP 400','Upstream HTTP 404'):break
|
|
result['errors'].append({'source':kind,'page':page,'error':str(exc)[:150]});break
|
|
except (ValueError,httpx.HTTPError) as exc:
|
|
result['errors'].append({'source':kind,'page':page,'error':str(exc)[:150]});break
|
|
if not content:break
|
|
for item in content:
|
|
try:
|
|
permalink=item.get('link') or ''
|
|
parsed=urlparse(permalink)
|
|
if not parsed.path:raise ValueError('未返回页面 URL')
|
|
if parsed.hostname!=urlparse(root).hostname:raise ValueError('返回了不同域名的链接，需手动映射')
|
|
route=route_path(parsed.path)
|
|
title=BeautifulSoup(item.get('title',{}).get('rendered',''),'html.parser').get_text(' ',strip=True)
|
|
excerpt=BeautifulSoup(item.get('excerpt',{}).get('rendered',''),'html.parser').get_text(' ',strip=True)
|
|
seo=item.get('yoast_head_json') or {}
|
|
data={'type':'post' if kind=='posts' else 'page', 'route':route,'title':title,
|
|
'summary':excerpt,'body':item.get('content',{}).get('rendered',''),
|
|
'seo_title':seo.get('title',''),'seo_description':seo.get('description',''),
|
|
'canonical':seo.get('canonical',''),'meta':{'wp_id':item.get('id'),'wp_slug':item.get('slug'),'wp_modified':item.get('modified')}}
|
|
previous=db.one('SELECT id FROM entries WHERE site_id=? AND route=?',(site_id,route))
|
|
if previous and on_conflict=='skip':result['skipped']+=1;continue
|
|
save_entry(site_id,data,existing_id=previous['id'] if previous else None,source='wp-rest')
|
|
result['updated' if previous else 'imported']+=1
|
|
except Exception as exc:
|
|
result['errors'].append({'item':item.get('id'),'error':str(exc)[:180]})
|
|
remaining-=1
|
|
if remaining<=0:break
|
|
if len(content)<batch_size:break
|
|
page+=1
|
|
return result
|