chore: baseline V8.10.0-dev.1 with v8_workflows type-mapping fix and trade-pro spec

- SiteForge Enterprise V8.10.0-dev.1 upstream source (210 tests) - Fix: v8_workflows._mapped_input now reads both 类型/分类 columns so WXR imports keep post/product types instead of collapsing to page - Docs: trade-pro theme port spec (R3) and implementation plan Co-Authored-By: Claude Code <noreply@anthropic.com>
2026-10-22 16:49:25 +08:00
commit 8cea7dee20
354 changed files with 20736 additions and 0 deletions
@@ -0,0 +1,228 @@
+"""Data-import adapters for CSV/JSON/XLSX/WordPress WXR/public HTML crawl."""
+import csv
+import io
+import json
+import re
+import time
+from urllib.parse import urlparse, urljoin, urldefrag
+from collections import deque
+from defusedxml import ElementTree as SafeET
+from bs4 import BeautifulSoup
+import httpx
+from openpyxl import load_workbook
+from .core import route_path, save_entry, ensure_public_url, slug_of
+from . import db,wordpress_compat,safe_http
+
+KEY_ALIASES={
+    'title':('title','name','标题','名称','产品名称','post_title'),
+    'type':('type','post_type','类型','内容类型'),
+    'route':('route','url','path','permalink','link','路径','链接'),
+    'summary':('summary','excerpt','description','摘要','简介'),
+    'body':('body','content','text','details','正文','内容'),
+    'cover':('cover','image','image_url','featured_image','图片','封面'),
+    'seo_title':('seo_title','meta_title','SEO标题'),
+    'seo_description':('seo_description','meta_description','SEO描述'),
+    'canonical':('canonical',),
+    'status':('status',),
+}
+
+def normalize(rec):
+    r={str(k).strip().lower():v for k,v in rec.items()}
+    clean={}
+    used=set()
+    for canonical,aliases in KEY_ALIASES.items():
+        for alias in aliases:
+            if alias.lower() in r:
+                clean[canonical]=r[alias.lower()] if r[alias.lower()] is not None else ''
+                used.add(alias.lower());break
+    extra={k:v for k,v in r.items() if k not in used}
+    embedded=r.get('meta') if isinstance(r.get('meta'),dict) else {}
+    clean['meta']={**embedded,**{k:v for k,v in extra.items() if k not in ('id','meta')}}
+    if not clean.get('type'):clean['type']='product'
+    return clean
+
+def import_records(site_id, records, origin='file', on_conflict='skip'):
+    if on_conflict not in ('skip','update'):
+        raise ValueError('无效的冲突策略')
+    result={'imported':0,'updated':0,'skipped':0,'errors':[]}
+    for index,rec in enumerate(records,1):
+        try:
+            data=normalize(rec)
+            if not data.get('title'):raise ValueError('缺少 title/name')
+            kind=data.get('type') or 'product'
+            if not data.get('route'):
+                data['route']=f'/{kind}/{slug_of(str(data["title"]))}/'
+            if '?' in str(data['route']) and not str(data['route']).startswith('http'):
+                raise ValueError('带查询参数的动态 URL 需要专项映射，不能直接转换为静态路由')
+            route=route_path(data['route'])
+            existing=db.one('SELECT id FROM entries WHERE site_id=? AND route=?',(site_id,route))
+            if existing and on_conflict=='skip':
+                result['skipped']+=1;continue
+            save_entry(site_id,data,existing_id=existing['id'] if existing else None,source=origin)
+            result['updated' if existing else 'imported']+=1
+        except Exception as exc:
+            if len(result['errors'])<80:
+                result['errors'].append({'row':index,'message':str(exc)[:250]})
+    return result
+
+def import_data_file(site_id, filename, content, conflict):
+    lower=filename.lower()
+    if lower.endswith('.csv'):
+        txt=content.decode('utf-8-sig')
+        records=list(csv.DictReader(io.StringIO(txt)))
+    elif lower.endswith('.json'):
+        parsed=json.loads(content.decode('utf-8-sig'))
+        if isinstance(parsed,dict):parsed=parsed.get('entries',parsed.get('items',[]))
+        if not isinstance(parsed,list):raise ValueError('JSON 应当为数据数组或包含 entries/items 数组')
+        records=parsed
+    elif lower.endswith('.xlsx'):
+        wb=load_workbook(io.BytesIO(content),read_only=True,data_only=True)
+        ws=wb.active
+        values=ws.iter_rows(values_only=True)
+        headers=[str(x or '').strip() for x in next(values)]
+        records=(dict(zip(headers,row)) for row in values)
+    elif lower.endswith('.xml'):
+        return import_wxr(site_id,content,conflict)
+    else:
+        raise ValueError('仅支持 .csv/.json/.xlsx/.xml')
+    return import_records(site_id,records,filename,conflict)
+
+def import_wxr(site_id,content,conflict='skip'):
+    root=SafeET.fromstring(content)
+    ns={'content':'http://purl.org/rss/1.0/modules/content/',
+        'excerpt':'http://wordpress.org/export/1.2/excerpt/',
+        'wp':'http://wordpress.org/export/1.2/'}
+    # Support alternate WXR versions (e.g. 1.1)
+    raw_content=None; raw_wp=None; raw_excerpt=None
+    for e in root.iter():
+        if e.tag.startswith('{'):
+            uri=e.tag.split('}')[0][1:]
+            if uri.endswith('/modules/content/'):raw_content=uri
+            if 'wordpress.org/export/' in uri and uri.endswith('/') :raw_wp=uri
+            if uri.endswith('/modules/excerpt/'):raw_excerpt=uri
+    if raw_content:ns['content']=raw_content
+    if raw_wp:ns['wp']=raw_wp
+    if raw_excerpt:ns['excerpt']=raw_excerpt
+    entries=[]
+    media_index=wordpress_compat.wp_media_map(root)
+    for it in root.findall('./channel/item'):
+        typ=(it.findtext('wp:post_type',default='post',namespaces=ns) or 'post')
+        status=(it.findtext('wp:status',default='publish',namespaces=ns) or 'publish')
+        if typ in ('attachment','nav_menu_item','revision') or status not in ('publish','private','draft'):
+            continue
+        slug=it.findtext('wp:post_name',namespaces=ns) or slug_of(it.findtext('title') or 'page')
+        title=it.findtext('title') or slug
+        link=it.findtext('link') or ''
+        path=urlparse(link).path if link.startswith(('http://','https://')) else ''
+        if not path: path=f'/{slug}/' if typ=='page' else f'/{typ}/{slug}/'
+        wpmeta={}
+        for pm in it.findall('wp:postmeta',ns):
+            key=pm.findtext('wp:meta_key',namespaces=ns)
+            if key:wpmeta[key]=pm.findtext('wp:meta_value',default='',namespaces=ns)
+        body=it.findtext('content:encoded',default='',namespaces=ns)
+        taxonomy=wordpress_compat.wp_taxonomy(it)
+        # ACF stores human-readable fields as plain postmeta keys, while
+        # underscore-prefixed field references point to the ACF field schema.
+        # Preserve *all* raw metadata, but expose only safe scalar values as
+        # structured content fields for the visual parameter component.
+        fields={k:v[:10000] for k,v in wpmeta.items()
+                if k and not k.startswith('_') and not k.lower().startswith('field_')
+                and isinstance(v,str) and len(v)<=10000}
+        for source,dest in (('_sku','sku'),('_regular_price','regular_price'),
+                            ('_sale_price','sale_price'),('_price','price'),
+                            ('_stock_status','stock_status'),('_weight','weight')):
+            if wpmeta.get(source):fields[dest]=str(wpmeta[source])[:1000]
+        fields.update(taxonomy)
+        wp_featured=media_index.get(wpmeta.get('_thumbnail_id',''),'')
+        gallery_ids=[x.strip() for x in wpmeta.get('_product_image_gallery','').split(',') if x.strip()]
+        gallery=[media_index[x] for x in gallery_ids if x in media_index]
+        if gallery:fields['gallery']=gallery
+        warnings=[]
+        if wpmeta.get('_elementor_data'):
+            elementor=wordpress_compat.elementor_to_html(wpmeta['_elementor_data'])
+            if not body.strip() and elementor['html']:
+                body=elementor['html'];warnings.append('Elementor 仅转换已支持的文本与图片组件，原始布局需人工检查')
+            warnings.extend(elementor['warnings'])
+            warnings.append('检测到 Elementor 页面构建器 JSON；原生布局不能直接转换为 SiteForge 区块，需人工对照检查。')
+        if any(k.startswith('_product_attributes') for k in wpmeta):
+            warnings.append('WooCommerce 序列化商品属性待专项解析，原始 metadata 已保留。')
+        body=body or ''
+        entries.append({'type':typ if typ in ('page','post','product','service','case','brand','vehicle') else 'other',
+          'title':title,'route':path,'body':body,'cover':wp_featured,
+          'summary':it.findtext('excerpt:encoded',default='',namespaces=ns),
+          'seo_title':wpmeta.get('_yoast_wpseo_title') or wpmeta.get('rank_math_title') or '',
+          'seo_description':wpmeta.get('_yoast_wpseo_metadesc') or wpmeta.get('rank_math_description') or '',
+          'canonical':wpmeta.get('_yoast_wpseo_canonical') or wpmeta.get('rank_math_canonical_url') or '',
+          'robots':'noindex,follow' if wpmeta.get('_yoast_wpseo_meta-robots-noindex')=='1' or wpmeta.get('rank_math_robots')=='noindex' else 'index,follow',
+          'status':'published' if status=='publish' else 'draft',
+          'meta':{'elementor_widgets':elementor['widgets'] if wpmeta.get('_elementor_data') else 0,'wp_post_id':it.findtext('wp:post_id',default='',namespaces=ns),'wp_type':typ,'wp_metadata':wpmeta,'fields':fields,'migration_warnings':warnings}})
+    return import_records(site_id,entries,'wordpress-wxr',conflict)
+
+def safe_get(client,url,max_bytes=3_000_000):
+    """Fetch within size and redirect limits using pinned public-only connections."""
+    response=safe_http.fetch(url,max_bytes=max_bytes,timeout=12,redirects=3)
+    response.raise_for_status()
+    if 'text/html' not in response.headers.get('content-type','text/html'):
+        raise ValueError('页面不是 HTML')
+    return response.content,response.url
+
+def extract_page(html,url):
+    s=BeautifulSoup(html,'html.parser')
+    title=s.title.get_text(' ',strip=True) if s.title else ''
+    description=s.find('meta',attrs={'name':re.compile('^description$',re.I)})
+    canonical=s.find('link',attrs={'rel':lambda val: bool(val) and ('canonical' in val if isinstance(val,list) else 'canonical' in val)})
+    robots=s.find('meta',attrs={'name':re.compile('^robots$',re.I)})
+    h1=s.find('h1')
+    heading=h1.get_text(' ',strip=True) if h1 else title
+    main=s.find('main') or s.find('article') or s.body
+    body=''
+    if main:
+        for tag in main.select('script,style,nav,footer,header,aside,form,iframe,svg'):
+            tag.decompose()
+        for image in main.find_all('img'):
+            src=image.get('src') or image.get('data-src')
+            if src:image['src']=urljoin(url,src)
+        for a in main.find_all('a',href=True):
+            if a['href'].startswith(('http','/','.')):a['href']=urljoin(url,a['href'])
+        body=str(main)
+    links=[]
+    for a in s.find_all('a',href=True):
+        abs_url=urljoin(url,a['href'])
+        if urlparse(abs_url).scheme in ('http','https'):
+            links.append(urldefrag(abs_url)[0])
+    return ({'type':'page' if urlparse(url).path=='/' else 'post',
+      'title':heading or 'Untitled','route':urlparse(url).path or '/',
+      'body':body,'summary':description.get('content','') if description else '',
+      'seo_title':title,'seo_description':description.get('content','') if description else '',
+      'canonical':canonical.get('href','') if canonical else '',
+      'robots':robots.get('content','index,follow') if robots else 'index,follow'},links)
+
+def crawl_site(site_id,start,max_pages=25,conflict='skip'):
+    ensure_public_url(start)
+    max_pages=max(1,min(int(max_pages),60))
+    root=urlparse(start)
+    host=root.hostname
+    queue=deque([start]);visited=set();pages=[];errors=[]
+    # Strict same-host traversal, not open web crawling.
+    with httpx.Client(timeout=httpx.Timeout(12,connect=5),headers={'User-Agent':'SiteForgeLocalMigration/0.1'},trust_env=False) as client:
+        while queue and len(visited)<max_pages:
+            url=queue.popleft()
+            if url in visited:continue
+            parsed=urlparse(url)
+            if parsed.hostname!=host:continue
+            if re.search(r'/(wp-admin|wp-login|cart|checkout|admin)(/|$)',parsed.path,re.I):continue
+            visited.add(url)
+            try:
+                data,actual=safe_get(client,url)
+                if urlparse(actual).hostname!=host:raise ValueError('跳转至站外地址')
+                page,links=extract_page(data,actual)
+                pages.append(page)
+                for link in links:
+                    if urlparse(link).hostname==host and link not in visited and link not in queue:
+                        if not re.search(r'\.(pdf|zip|jpg|jpeg|png|gif|webp|js|css)(\?|$)',link,re.I):
+                            queue.append(link)
+            except Exception as exc:
+                if len(errors)<40:errors.append({'url':url,'error':str(exc)[:180]})
+    result=import_records(site_id,pages,'crawler',conflict)
+    result.update({'crawled':len(visited),'discovered':len(pages),'fetch_errors':errors})
+    return result