"""Data-import adapters for CSV/JSON/XLSX/WordPress WXR/public HTML crawl.""" import csv import io import json import re import time from urllib.parse import urlparse, urljoin, urldefrag from collections import deque from defusedxml import ElementTree as SafeET from bs4 import BeautifulSoup import httpx from openpyxl import load_workbook from .core import route_path, save_entry, ensure_public_url, slug_of from . import db,wordpress_compat,safe_http,wp_complex_v820 KEY_ALIASES={ 'title':('title','name','标题','名称','产品名称','post_title'), 'type':('type','post_type','类型','内容类型'), 'route':('route','url','path','permalink','link','路径','链接'), 'summary':('summary','excerpt','description','摘要','简介'), 'body':('body','content','text','details','正文','内容'), 'cover':('cover','image','image_url','featured_image','图片','封面'), 'seo_title':('seo_title','meta_title','SEO标题'), 'seo_description':('seo_description','meta_description','SEO描述'), 'canonical':('canonical',), 'status':('status',), } def normalize(rec): r={str(k).strip().lower():v for k,v in rec.items()} clean={} used=set() for canonical,aliases in KEY_ALIASES.items(): for alias in aliases: if alias.lower() in r: clean[canonical]=r[alias.lower()] if r[alias.lower()] is not None else '' used.add(alias.lower());break extra={k:v for k,v in r.items() if k not in used} embedded=r.get('meta') if isinstance(r.get('meta'),dict) else {} clean['meta']={**embedded,**{k:v for k,v in extra.items() if k not in ('id','meta')}} if not clean.get('type'):clean['type']='product' return clean def import_records(site_id, records, origin='file', on_conflict='skip'): if on_conflict not in ('skip','update'): raise ValueError('无效的冲突策略') result={'imported':0,'updated':0,'skipped':0,'errors':[]} for index,rec in enumerate(records,1): try: data=normalize(rec) if not data.get('title'):raise ValueError('缺少 title/name') kind=data.get('type') or 'product' if not data.get('route'): data['route']=f'/{kind}/{slug_of(str(data["title"]))}/' if '?' in str(data['route']) and not str(data['route']).startswith('http'): raise ValueError('带查询参数的动态 URL 需要专项映射，不能直接转换为静态路由') route=route_path(data['route']) existing=db.one('SELECT id FROM entries WHERE site_id=? AND route=?',(site_id,route)) if existing and on_conflict=='skip': result['skipped']+=1;continue save_entry(site_id,data,existing_id=existing['id'] if existing else None,source=origin) result['updated' if existing else 'imported']+=1 except Exception as exc: if len(result['errors'])<80: result['errors'].append({'row':index,'message':str(exc)[:250]}) return result def import_data_file(site_id, filename, content, conflict): lower=filename.lower() if lower.endswith('.csv'): txt=content.decode('utf-8-sig') records=list(csv.DictReader(io.StringIO(txt))) elif lower.endswith('.json'): parsed=json.loads(content.decode('utf-8-sig')) if isinstance(parsed,dict):parsed=parsed.get('entries',parsed.get('items',[])) if not isinstance(parsed,list):raise ValueError('JSON 应当为数据数组或包含 entries/items 数组') records=parsed elif lower.endswith('.xlsx'): wb=load_workbook(io.BytesIO(content),read_only=True,data_only=True) ws=wb.active values=ws.iter_rows(values_only=True) headers=[str(x or '').strip() for x in next(values)] records=(dict(zip(headers,row)) for row in values) elif lower.endswith('.xml'): return import_wxr(site_id,content,conflict) else: raise ValueError('仅支持 .csv/.json/.xlsx/.xml') return import_records(site_id,records,filename,conflict) def import_wxr(site_id,content,conflict='skip'): root=SafeET.fromstring(content) ns={'content':'http://purl.org/rss/1.0/modules/content/', 'excerpt':'http://wordpress.org/export/1.2/excerpt/', 'wp':'http://wordpress.org/export/1.2/'} # Support alternate WXR versions (e.g. 1.1) raw_content=None; raw_wp=None; raw_excerpt=None for e in root.iter(): if e.tag.startswith('{'): uri=e.tag.split('}')[0][1:] if uri.endswith('/modules/content/'):raw_content=uri if 'wordpress.org/export/' in uri and uri.endswith('/') :raw_wp=uri if uri.endswith('/modules/excerpt/'):raw_excerpt=uri if raw_content:ns['content']=raw_content if raw_wp:ns['wp']=raw_wp if raw_excerpt:ns['excerpt']=raw_excerpt entries=[] media_index=wordpress_compat.wp_media_map(root) for it in root.findall('./channel/item'): typ=(it.findtext('wp:post_type',default='post',namespaces=ns) or 'post') status=(it.findtext('wp:status',default='publish',namespaces=ns) or 'publish') if typ in ('attachment','nav_menu_item','revision') or status not in ('publish','private','draft'): continue slug=it.findtext('wp:post_name',namespaces=ns) or slug_of(it.findtext('title') or 'page') title=it.findtext('title') or slug link=it.findtext('link') or '' path=urlparse(link).path if link.startswith(('http://','https://')) else '' if urlparse(link).query or urlparse(link).fragment: raise ValueError(f'WXR 记录 {title[:80]} 使用动态参数 URL；请先做 V8.20 预检和显式路由映射') if not path: raise ValueError(f'WXR 记录 {title[:80]} 缺少原始 URL，无法保证旧路径不变') wpmeta={} for pm in it.findall('wp:postmeta',ns): key=pm.findtext('wp:meta_key',namespaces=ns) if key:wpmeta[key]=pm.findtext('wp:meta_value',default='',namespaces=ns) body=it.findtext('content:encoded',default='',namespaces=ns) taxonomy=wordpress_compat.wp_taxonomy(it) # ACF stores human-readable fields as plain postmeta keys, while # underscore-prefixed field references point to the ACF field schema. # Preserve *all* raw metadata, but expose only safe scalar values as # structured content fields for the visual parameter component. fields={k:v[:10000] for k,v in wpmeta.items() if k and not k.startswith('_') and not k.lower().startswith('field_') and isinstance(v,str) and len(v)<=10000} for source,dest in (('_sku','sku'),('_regular_price','regular_price'), ('_sale_price','sale_price'),('_price','price'), ('_stock_status','stock_status'),('_weight','weight')): if wpmeta.get(source):fields[dest]=str(wpmeta[source])[:1000] fields.update(taxonomy) acf=wp_complex_v820.extract_acf(wpmeta) for key,value in acf['fields'].items(): fields.setdefault(key,value) for key,value in acf['repeaters'].items(): fields[key]=value wp_featured=media_index.get(wpmeta.get('_thumbnail_id',''),'') gallery_ids=[x.strip() for x in wpmeta.get('_product_image_gallery','').split(',') if x.strip()] gallery=[media_index[x] for x in gallery_ids if x in media_index] if gallery:fields['gallery']=gallery warnings=list(acf['warnings']) if wpmeta.get('_elementor_data'): elementor=wordpress_compat.elementor_to_html(wpmeta['_elementor_data']) if not body.strip() and elementor['html']: body=elementor['html'];warnings.append('Elementor 仅转换已支持的文本与图片组件，原始布局需人工检查') warnings.extend(elementor['warnings']) warnings.append('检测到 Elementor 页面构建器 JSON；原生布局不能直接转换为 SiteForge 区块，需人工对照检查。') if any(k.startswith('_product_attributes') for k in wpmeta): warnings.append('WooCommerce 序列化商品属性待专项解析，原始 metadata 已保留。') body=body or '' entries.append({'type':typ if typ in ('page','post','product','service','case','brand','vehicle') else 'other', 'title':title,'route':path,'body':body,'cover':wp_featured, 'summary':it.findtext('excerpt:encoded',default='',namespaces=ns), 'seo_title':wpmeta.get('_yoast_wpseo_title') or wpmeta.get('rank_math_title') or '', 'seo_description':wpmeta.get('_yoast_wpseo_metadesc') or wpmeta.get('rank_math_description') or '', 'canonical':wpmeta.get('_yoast_wpseo_canonical') or wpmeta.get('rank_math_canonical_url') or '', 'robots':'noindex,follow' if wpmeta.get('_yoast_wpseo_meta-robots-noindex')=='1' or wpmeta.get('rank_math_robots')=='noindex' else 'index,follow', 'status':'published' if status=='publish' else 'draft', 'meta':{'elementor_widgets':elementor['widgets'] if wpmeta.get('_elementor_data') else 0,'wp_post_id':it.findtext('wp:post_id',default='',namespaces=ns),'wp_type':typ,'wp_metadata':wpmeta,'fields':fields,'acf_repeaters':acf['repeaters'],'migration_warnings':warnings}}) return import_records(site_id,entries,'wordpress-wxr',conflict) def safe_get(client,url,max_bytes=3_000_000): """Fetch within size and redirect limits using pinned public-only connections.""" response=safe_http.fetch(url,max_bytes=max_bytes,timeout=12,redirects=3) response.raise_for_status() if 'text/html' not in response.headers.get('content-type','text/html'): raise ValueError('页面不是 HTML') return response.content,response.url def extract_page(html,url): s=BeautifulSoup(html,'html.parser') title=s.title.get_text(' ',strip=True) if s.title else '' description=s.find('meta',attrs={'name':re.compile('^description$',re.I)}) canonical=s.find('link',attrs={'rel':lambda val: bool(val) and ('canonical' in val if isinstance(val,list) else 'canonical' in val)}) robots=s.find('meta',attrs={'name':re.compile('^robots$',re.I)}) h1=s.find('h1') heading=h1.get_text(' ',strip=True) if h1 else title main=s.find('main') or s.find('article') or s.body body='' if main: for tag in main.select('script,style,nav,footer,header,aside,form,iframe,svg'): tag.decompose() for image in main.find_all('img'): src=image.get('src') or image.get('data-src') if src:image['src']=urljoin(url,src) for a in main.find_all('a',href=True): if a['href'].startswith(('http','/','.')):a['href']=urljoin(url,a['href']) body=str(main) links=[] for a in s.find_all('a',href=True): abs_url=urljoin(url,a['href']) if urlparse(abs_url).scheme in ('http','https'): links.append(urldefrag(abs_url)[0]) return ({'type':'page' if urlparse(url).path=='/' else 'post', 'title':heading or 'Untitled','route':urlparse(url).path or '/', 'body':body,'summary':description.get('content','') if description else '', 'seo_title':title,'seo_description':description.get('content','') if description else '', 'canonical':canonical.get('href','') if canonical else '', 'robots':robots.get('content','index,follow') if robots else 'index,follow'},links) def crawl_site(site_id,start,max_pages=25,conflict='skip'): ensure_public_url(start) max_pages=max(1,min(int(max_pages),60)) root=urlparse(start) host=root.hostname queue=deque([start]);visited=set();pages=[];errors=[] # Strict same-host traversal, not open web crawling. with httpx.Client(timeout=httpx.Timeout(12,connect=5),headers={'User-Agent':'SiteForgeLocalMigration/0.1'},trust_env=False) as client: while queue and len(visited)