浅念
3c57c167f2
导入 SiteForge Factory V5 基线源码： - FastAPI + SQLite + Docker Compose（Web + Worker）架构 - 四套商业主题 × 14 页面模板（1.1.0，含历史版本快照） - 四种建站模式骨架 + 迁移审计 + 发布门禁 - 52 项 pytest 测试 - 补全计划主文档 docs/superpowers/plans/ Co-Authored-By: Claude Code <noreply@anthropic.com>
240 lines
13 KiB
Python
240 lines
13 KiB
Python
"""Data-import adapters for CSV/JSON/XLSX/WordPress WXR/public HTML crawl."""
|
|
import csv
|
|
import io
|
|
import json
|
|
import re
|
|
import time
|
|
from urllib.parse import urlparse, urljoin, urldefrag
|
|
from collections import deque
|
|
from defusedxml import ElementTree as SafeET
|
|
from bs4 import BeautifulSoup
|
|
import httpx
|
|
from openpyxl import load_workbook
|
|
from .core import route_path, save_entry, ensure_public_url, slug_of
|
|
from . import db,wordpress_compat
|
|
|
|
KEY_ALIASES={
|
|
'title':('title','name','标题','名称','产品名称','post_title'),
|
|
'type':('type','post_type','类型','内容类型'),
|
|
'route':('route','url','path','permalink','link','路径','链接'),
|
|
'summary':('summary','excerpt','description','摘要','简介'),
|
|
'body':('body','content','text','details','正文','内容'),
|
|
'cover':('cover','image','image_url','featured_image','图片','封面'),
|
|
'seo_title':('seo_title','meta_title','SEO标题'),
|
|
'seo_description':('seo_description','meta_description','SEO描述'),
|
|
'canonical':('canonical',),
|
|
'status':('status',),
|
|
}
|
|
|
|
def normalize(rec):
|
|
r={str(k).strip().lower():v for k,v in rec.items()}
|
|
clean={}
|
|
used=set()
|
|
for canonical,aliases in KEY_ALIASES.items():
|
|
for alias in aliases:
|
|
if alias.lower() in r:
|
|
clean[canonical]=r[alias.lower()] if r[alias.lower()] is not None else ''
|
|
used.add(alias.lower());break
|
|
extra={k:v for k,v in r.items() if k not in used}
|
|
embedded=r.get('meta') if isinstance(r.get('meta'),dict) else {}
|
|
clean['meta']={**embedded,**{k:v for k,v in extra.items() if k not in ('id','meta')}}
|
|
if not clean.get('type'):clean['type']='product'
|
|
return clean
|
|
|
|
def import_records(site_id, records, origin='file', on_conflict='skip'):
|
|
if on_conflict not in ('skip','update'):
|
|
raise ValueError('无效的冲突策略')
|
|
result={'imported':0,'updated':0,'skipped':0,'errors':[]}
|
|
for index,rec in enumerate(records,1):
|
|
try:
|
|
data=normalize(rec)
|
|
if not data.get('title'):raise ValueError('缺少 title/name')
|
|
kind=data.get('type') or 'product'
|
|
if not data.get('route'):
|
|
data['route']=f'/{kind}/{slug_of(str(data["title"]))}/'
|
|
if '?' in str(data['route']) and not str(data['route']).startswith('http'):
|
|
raise ValueError('带查询参数的动态 URL 需要专项映射，不能直接转换为静态路由')
|
|
route=route_path(data['route'])
|
|
existing=db.one('SELECT id FROM entries WHERE site_id=? AND route=?',(site_id,route))
|
|
if existing and on_conflict=='skip':
|
|
result['skipped']+=1;continue
|
|
save_entry(site_id,data,existing_id=existing['id'] if existing else None,source=origin)
|
|
result['updated' if existing else 'imported']+=1
|
|
except Exception as exc:
|
|
if len(result['errors'])<80:
|
|
result['errors'].append({'row':index,'message':str(exc)[:250]})
|
|
return result
|
|
|
|
def import_data_file(site_id, filename, content, conflict):
|
|
lower=filename.lower()
|
|
if lower.endswith('.csv'):
|
|
txt=content.decode('utf-8-sig')
|
|
records=list(csv.DictReader(io.StringIO(txt)))
|
|
elif lower.endswith('.json'):
|
|
parsed=json.loads(content.decode('utf-8-sig'))
|
|
if isinstance(parsed,dict):parsed=parsed.get('entries',parsed.get('items',[]))
|
|
if not isinstance(parsed,list):raise ValueError('JSON 应当为数据数组或包含 entries/items 数组')
|
|
records=parsed
|
|
elif lower.endswith('.xlsx'):
|
|
wb=load_workbook(io.BytesIO(content),read_only=True,data_only=True)
|
|
ws=wb.active
|
|
values=ws.iter_rows(values_only=True)
|
|
headers=[str(x or '').strip() for x in next(values)]
|
|
records=(dict(zip(headers,row)) for row in values)
|
|
elif lower.endswith('.xml'):
|
|
return import_wxr(site_id,content,conflict)
|
|
else:
|
|
raise ValueError('仅支持 .csv/.json/.xlsx/.xml')
|
|
return import_records(site_id,records,filename,conflict)
|
|
|
|
def import_wxr(site_id,content,conflict='skip'):
|
|
root=SafeET.fromstring(content)
|
|
ns={'content':'http://purl.org/rss/1.0/modules/content/',
|
|
'excerpt':'http://wordpress.org/export/1.2/excerpt/',
|
|
'wp':'http://wordpress.org/export/1.2/'}
|
|
# Support alternate WXR versions (e.g. 1.1)
|
|
raw_content=None; raw_wp=None; raw_excerpt=None
|
|
for e in root.iter():
|
|
if e.tag.startswith('{'):
|
|
uri=e.tag.split('}')[0][1:]
|
|
if uri.endswith('/modules/content/'):raw_content=uri
|
|
if 'wordpress.org/export/' in uri and uri.endswith('/') :raw_wp=uri
|
|
if uri.endswith('/modules/excerpt/'):raw_excerpt=uri
|
|
if raw_content:ns['content']=raw_content
|
|
if raw_wp:ns['wp']=raw_wp
|
|
if raw_excerpt:ns['excerpt']=raw_excerpt
|
|
entries=[]
|
|
media_index=wordpress_compat.wp_media_map(root)
|
|
for it in root.findall('./channel/item'):
|
|
typ=(it.findtext('wp:post_type',default='post',namespaces=ns) or 'post')
|
|
status=(it.findtext('wp:status',default='publish',namespaces=ns) or 'publish')
|
|
if typ in ('attachment','nav_menu_item','revision') or status not in ('publish','private','draft'):
|
|
continue
|
|
slug=it.findtext('wp:post_name',namespaces=ns) or slug_of(it.findtext('title') or 'page')
|
|
title=it.findtext('title') or slug
|
|
link=it.findtext('link') or ''
|
|
path=urlparse(link).path if link.startswith(('http://','https://')) else ''
|
|
if not path: path=f'/{slug}/' if typ=='page' else f'/{typ}/{slug}/'
|
|
wpmeta={}
|
|
for pm in it.findall('wp:postmeta',ns):
|
|
key=pm.findtext('wp:meta_key',namespaces=ns)
|
|
if key:wpmeta[key]=pm.findtext('wp:meta_value',default='',namespaces=ns)
|
|
body=it.findtext('content:encoded',default='',namespaces=ns)
|
|
taxonomy=wordpress_compat.wp_taxonomy(it)
|
|
# ACF stores human-readable fields as plain postmeta keys, while
|
|
# underscore-prefixed field references point to the ACF field schema.
|
|
# Preserve *all* raw metadata, but expose only safe scalar values as
|
|
# structured content fields for the visual parameter component.
|
|
fields={k:v[:10000] for k,v in wpmeta.items()
|
|
if k and not k.startswith('_') and not k.lower().startswith('field_')
|
|
and isinstance(v,str) and len(v)<=10000}
|
|
for source,dest in (('_sku','sku'),('_regular_price','regular_price'),
|
|
('_sale_price','sale_price'),('_price','price'),
|
|
('_stock_status','stock_status'),('_weight','weight')):
|
|
if wpmeta.get(source):fields[dest]=str(wpmeta[source])[:1000]
|
|
fields.update(taxonomy)
|
|
wp_featured=media_index.get(wpmeta.get('_thumbnail_id',''),'')
|
|
gallery_ids=[x.strip() for x in wpmeta.get('_product_image_gallery','').split(',') if x.strip()]
|
|
gallery=[media_index[x] for x in gallery_ids if x in media_index]
|
|
if gallery:fields['gallery']=gallery
|
|
warnings=[]
|
|
if wpmeta.get('_elementor_data'):
|
|
elementor=wordpress_compat.elementor_to_html(wpmeta['_elementor_data'])
|
|
if not body.strip() and elementor['html']:
|
|
body=elementor['html'];warnings.append('Elementor 仅转换已支持的文本与图片组件，原始布局需人工检查')
|
|
warnings.extend(elementor['warnings'])
|
|
warnings.append('检测到 Elementor 页面构建器 JSON；原生布局不能直接转换为 SiteForge 区块，需人工对照检查。')
|
|
if any(k.startswith('_product_attributes') for k in wpmeta):
|
|
warnings.append('WooCommerce 序列化商品属性待专项解析，原始 metadata 已保留。')
|
|
body=body or ''
|
|
entries.append({'type':typ if typ in ('page','post','product','service','case','brand','vehicle') else 'other',
|
|
'title':title,'route':path,'body':body,'cover':wp_featured,
|
|
'summary':it.findtext('excerpt:encoded',default='',namespaces=ns),
|
|
'seo_title':wpmeta.get('_yoast_wpseo_title') or wpmeta.get('rank_math_title') or '',
|
|
'seo_description':wpmeta.get('_yoast_wpseo_metadesc') or wpmeta.get('rank_math_description') or '',
|
|
'canonical':wpmeta.get('_yoast_wpseo_canonical') or wpmeta.get('rank_math_canonical_url') or '',
|
|
'robots':'noindex,follow' if wpmeta.get('_yoast_wpseo_meta-robots-noindex')=='1' or wpmeta.get('rank_math_robots')=='noindex' else 'index,follow',
|
|
'status':'published' if status=='publish' else 'draft',
|
|
'meta':{'elementor_widgets':elementor['widgets'] if wpmeta.get('_elementor_data') else 0,'wp_post_id':it.findtext('wp:post_id',default='',namespaces=ns),'wp_type':typ,'wp_metadata':wpmeta,'fields':fields,'migration_warnings':warnings}})
|
|
return import_records(site_id,entries,'wordpress-wxr',conflict)
|
|
|
|
def safe_get(client,url,max_bytes=3_000_000):
|
|
"""Validate each hop to prevent trivial redirect-to-private host SSRF."""
|
|
for _ in range(4):
|
|
ensure_public_url(url)
|
|
with client.stream('GET',url, follow_redirects=False) as response:
|
|
if response.status_code in (301,302,303,307,308):
|
|
loc=response.headers.get('location')
|
|
if not loc: raise ValueError('重定向缺少 Location')
|
|
url=urljoin(url,loc);continue
|
|
response.raise_for_status()
|
|
if 'text/html' not in response.headers.get('content-type','text/html'):
|
|
raise ValueError('页面不是 HTML')
|
|
buff=bytearray()
|
|
for part in response.iter_bytes():
|
|
buff.extend(part)
|
|
if len(buff)>max_bytes:raise ValueError('HTML 页面超过 3 MB')
|
|
return bytes(buff),str(response.url)
|
|
raise ValueError('页面重定向次数过多')
|
|
|
|
def extract_page(html,url):
|
|
s=BeautifulSoup(html,'html.parser')
|
|
title=s.title.get_text(' ',strip=True) if s.title else ''
|
|
description=s.find('meta',attrs={'name':re.compile('^description$',re.I)})
|
|
canonical=s.find('link',attrs={'rel':lambda val: bool(val) and ('canonical' in val if isinstance(val,list) else 'canonical' in val)})
|
|
robots=s.find('meta',attrs={'name':re.compile('^robots$',re.I)})
|
|
h1=s.find('h1')
|
|
heading=h1.get_text(' ',strip=True) if h1 else title
|
|
main=s.find('main') or s.find('article') or s.body
|
|
body=''
|
|
if main:
|
|
for tag in main.select('script,style,nav,footer,header,aside,form,iframe,svg'):
|
|
tag.decompose()
|
|
for image in main.find_all('img'):
|
|
src=image.get('src') or image.get('data-src')
|
|
if src:image['src']=urljoin(url,src)
|
|
for a in main.find_all('a',href=True):
|
|
if a['href'].startswith(('http','/','.')):a['href']=urljoin(url,a['href'])
|
|
body=str(main)
|
|
links=[]
|
|
for a in s.find_all('a',href=True):
|
|
abs_url=urljoin(url,a['href'])
|
|
if urlparse(abs_url).scheme in ('http','https'):
|
|
links.append(urldefrag(abs_url)[0])
|
|
return ({'type':'page' if urlparse(url).path=='/' else 'post',
|
|
'title':heading or 'Untitled','route':urlparse(url).path or '/',
|
|
'body':body,'summary':description.get('content','') if description else '',
|
|
'seo_title':title,'seo_description':description.get('content','') if description else '',
|
|
'canonical':canonical.get('href','') if canonical else '',
|
|
'robots':robots.get('content','index,follow') if robots else 'index,follow'},links)
|
|
|
|
def crawl_site(site_id,start,max_pages=25,conflict='skip'):
|
|
ensure_public_url(start)
|
|
max_pages=max(1,min(int(max_pages),60))
|
|
root=urlparse(start)
|
|
host=root.hostname
|
|
queue=deque([start]);visited=set();pages=[];errors=[]
|
|
# Strict same-host traversal, not open web crawling.
|
|
with httpx.Client(timeout=httpx.Timeout(12,connect=5),headers={'User-Agent':'SiteForgeLocalMigration/0.1'},trust_env=False) as client:
|
|
while queue and len(visited)<max_pages:
|
|
url=queue.popleft()
|
|
if url in visited:continue
|
|
parsed=urlparse(url)
|
|
if parsed.hostname!=host:continue
|
|
if re.search(r'/(wp-admin|wp-login|cart|checkout|admin)(/|$)',parsed.path,re.I):continue
|
|
visited.add(url)
|
|
try:
|
|
data,actual=safe_get(client,url)
|
|
if urlparse(actual).hostname!=host:raise ValueError('跳转至站外地址')
|
|
page,links=extract_page(data,actual)
|
|
pages.append(page)
|
|
for link in links:
|
|
if urlparse(link).hostname==host and link not in visited and link not in queue:
|
|
if not re.search(r'\.(pdf|zip|jpg|jpeg|png|gif|webp|js|css)(\?|$)',link,re.I):
|
|
queue.append(link)
|
|
except Exception as exc:
|
|
if len(errors)<40:errors.append({'url':url,'error':str(exc)[:180]})
|
|
result=import_records(site_id,pages,'crawler',conflict)
|
|
result.update({'crawled':len(visited),'discovered':len(pages),'fetch_errors':errors})
|
|
return result
|