"""CMS adapter framework: bounded public-site detection and WP REST import. Private/paid/SSO content is intentionally not accessed by this adapter. Other platforms should provide a compatible adapter for their authorised APIs. """ from dataclasses import dataclass from urllib.parse import urljoin,urlparse from bs4 import BeautifulSoup import httpx from .core import ensure_public_url, route_path, save_entry from . import db, safe_http USER_AGENT='SiteForge/1.0 (+authorized-migration)' @dataclass(frozen=True) class SiteAnalysis: url:str platform:str title:str rest_api:str|None hints:list[str] def _get(client,url): # TCP is pinned to the public IP returned by the security validation, # while HTTPS still validates against the original hostname. response=safe_http.fetch(url,max_bytes=4_000_000,timeout=13,redirects=0) response.raise_for_status() return response def analyze(url): ensure_public_url(url) with httpx.Client(timeout=httpx.Timeout(13,connect=5),trust_env=False,headers={'User-Agent':USER_AGENT}) as client: r=_get(client,url) soup=BeautifulSoup(r.text,'html.parser') generator=soup.find('meta',attrs={'name':'generator'}) gen=(generator.get('content','') if generator else '').lower() rest=soup.find('link',rel='https://api.w.org/') api_url=rest.get('href') if rest else None platform='WordPress' if 'wordpress' in gen or api_url or 'wp-content/' in r.text[:200000] else ('Drupal' if 'drupal' in gen else 'Joomla' if 'joomla' in gen else 'Unknown / Custom') hints=[] if soup.find('meta',attrs={'name':'robots'}):hints.append('检测到 robots meta') if soup.find('link',rel='canonical'):hints.append('检测到 Canonical') if soup.find('script',attrs={'type':'application/ld+json'}):hints.append('检测到结构化数据') if not api_url and platform=='WordPress':api_url=urljoin(url,'/wp-json/') return SiteAnalysis(url,platform,soup.title.get_text(' ',strip=True)[:200] if soup.title else '',api_url,hints).__dict__ def import_wordpress_rest(site_id,origin,max_items=150,on_conflict='skip'): """Import public published posts/pages from a WP REST API, preserving URL paths. Special Elementor/ACF/Yoast fields require additional adapter support. """ if max_items<1 or max_items>1000:raise ValueError('允许导入 1-1000 条公开内容') if on_conflict not in ('skip','update'):raise ValueError('无效的冲突策略') root=origin.rstrip('/')+'/' ensure_public_url(root) result={'imported':0,'updated':0,'skipped':0,'errors':[]} with httpx.Client(timeout=httpx.Timeout(18,connect=6),trust_env=False,headers={'User-Agent':USER_AGENT}) as client: for kind in ('pages','posts'): batch_size=min(100,max_items) remaining=max_items-(result['imported']+result['updated']+result['skipped']) if remaining<=0:break page=1 while remaining>0 and page<=20: api=urljoin(root,f'wp-json/wp/v2/{kind}?per_page={batch_size}&page={page}&_embed=1') try: response=_get(client,api) content=response.json() if not isinstance(content,list):raise ValueError('API 返回内容不是数组') except safe_http.OutboundError as exc: if str(exc) in ('Upstream HTTP 400','Upstream HTTP 404'):break result['errors'].append({'source':kind,'page':page,'error':str(exc)[:150]});break except (ValueError,httpx.HTTPError) as exc: result['errors'].append({'source':kind,'page':page,'error':str(exc)[:150]});break if not content:break for item in content: try: permalink=item.get('link') or '' parsed=urlparse(permalink) if not parsed.path:raise ValueError('未返回页面 URL') if parsed.hostname!=urlparse(root).hostname:raise ValueError('返回了不同域名的链接,需手动映射') route=route_path(parsed.path) title=BeautifulSoup(item.get('title',{}).get('rendered',''),'html.parser').get_text(' ',strip=True) excerpt=BeautifulSoup(item.get('excerpt',{}).get('rendered',''),'html.parser').get_text(' ',strip=True) seo=item.get('yoast_head_json') or {} data={'type':'post' if kind=='posts' else 'page', 'route':route,'title':title, 'summary':excerpt,'body':item.get('content',{}).get('rendered',''), 'seo_title':seo.get('title',''),'seo_description':seo.get('description',''), 'canonical':seo.get('canonical',''),'meta':{'wp_id':item.get('id'),'wp_slug':item.get('slug'),'wp_modified':item.get('modified')}} previous=db.one('SELECT id FROM entries WHERE site_id=? AND route=?',(site_id,route)) if previous and on_conflict=='skip':result['skipped']+=1;continue save_entry(site_id,data,existing_id=previous['id'] if previous else None,source='wp-rest') result['updated' if previous else 'imported']+=1 except Exception as exc: result['errors'].append({'item':item.get('id'),'error':str(exc)[:180]}) remaining-=1 if remaining<=0:break if len(content)