127 lines
6.6 KiB
Python
127 lines
6.6 KiB
Python
|
"""Declarative mapping from arbitrary tabular source data to SiteForge content.
|
||
|
|
Never eval mappings, no AI hallucinated fields, preview shows all errors.
|
||
|
|
"""
|
||
|
|
import json,re
|
||
|
|
from . import db
|
||
|
|
from .core import route_path,slug_of,save_entry
|
||
|
|
|
||
|
|
ALIASES={
|
||
|
|
'title':('title','name','product_name','model_name','名称','标题','产品名称','车型','车型名称','商品名称'),
|
||
|
|
'route':('route','url','path','permalink','链接','路径','旧站链接','原网址'),
|
||
|
|
'summary':('summary','description','excerpt','简介','摘要','产品简介'),
|
||
|
|
'body':('body','content','html','正文','内容','详细介绍'),
|
||
|
|
'cover':('cover','image','thumbnail','featured_image','图片','封面','主图'),
|
||
|
|
'brand':('brand','manufacturer','品牌','厂商'),
|
||
|
|
'category':('category','分类','类目'),
|
||
|
|
'seo_title':('seo_title','meta_title','SEO标题','seo标题'),
|
||
|
|
'seo_description':('seo_description','meta_description','SEO描述','seo描述'),
|
||
|
|
'canonical':('canonical','规范网址'),
|
||
|
|
'model':('model','model_name','车型型号','型号','规格型号'),
|
||
|
|
'energy':('energy','energy_type','fuel_type','动力形式','能源类型','燃料类型'),
|
||
|
|
'range':('range','battery_range','纯电续航','续航','续航里程'),
|
||
|
|
'year':('year','model_year','年款','年份'),
|
||
|
|
'sku':('sku','product_sku','产品编号','货号'),
|
||
|
|
'gallery':('gallery','images','image_gallery','产品图片','图库','详情图片'),
|
||
|
|
'price':('price','msrp','售价','价格'),
|
||
|
|
'specs':('specs','attributes','parameters','技术参数','产品参数'),
|
||
|
|
|
||
|
|
}
|
||
|
|
SYSTEM_FIELDS={'title','route','summary','body','cover','seo_title','seo_description','canonical'}
|
||
|
|
|
||
|
|
|
||
|
|
def setup():
|
||
|
|
with db.connect() as c:
|
||
|
|
c.executescript('''CREATE TABLE IF NOT EXISTS mapping_profiles (
|
||
|
|
id TEXT PRIMARY KEY,site_id TEXT NOT NULL REFERENCES sites(id) ON DELETE CASCADE,
|
||
|
|
name TEXT NOT NULL,content_type TEXT NOT NULL, rules_json TEXT NOT NULL,
|
||
|
|
created_at TEXT NOT NULL,updated_at TEXT NOT NULL);
|
||
|
|
CREATE TABLE IF NOT EXISTS import_batches (
|
||
|
|
id TEXT PRIMARY KEY,site_id TEXT NOT NULL REFERENCES sites(id) ON DELETE CASCADE,
|
||
|
|
profile_id TEXT, source TEXT NOT NULL, result_json TEXT NOT NULL,
|
||
|
|
created_at TEXT NOT NULL);
|
||
|
|
''')
|
||
|
|
|
||
|
|
|
||
|
|
def suggest(columns):
|
||
|
|
clean=lambda x:re.sub(r'[\s\-_]+','',str(x).casefold())
|
||
|
|
src={clean(col):col for col in columns}
|
||
|
|
mapping={}
|
||
|
|
for target,aliases in ALIASES.items():
|
||
|
|
for alias in aliases:
|
||
|
|
if clean(alias) in src:
|
||
|
|
mapping[target]=src[clean(alias)];break
|
||
|
|
return mapping
|
||
|
|
|
||
|
|
|
||
|
|
def validate_rules(rules,columns):
|
||
|
|
if not isinstance(rules,dict) or len(rules)>100:raise ValueError('字段映射规则不合法')
|
||
|
|
for dest,src in rules.items():
|
||
|
|
if not re.fullmatch(r'[\w\-]{1,60}',dest,re.UNICODE):raise ValueError('目标字段名称不合法')
|
||
|
|
if not isinstance(src,str) or src not in columns:raise ValueError(f'来源字段不存在: {src}')
|
||
|
|
if 'title' not in rules:raise ValueError('必须映射标题字段')
|
||
|
|
|
||
|
|
|
||
|
|
def transform(rows,rules,kind='product',prefix=None):
|
||
|
|
if kind not in ('product','post','page','brand','vehicle','service','case','other'):raise ValueError('不支持的内容类型')
|
||
|
|
columns=set().union(*(r.keys() for r in rows)) if rows else set()
|
||
|
|
validate_rules(rules,columns)
|
||
|
|
seen=set();out=[];errors=[]
|
||
|
|
for index,row in enumerate(rows):
|
||
|
|
vals={}
|
||
|
|
for target,src in rules.items():
|
||
|
|
raw=row.get(src)
|
||
|
|
vals[target]=(json.dumps(raw,ensure_ascii=False) if isinstance(raw,(dict,list)) else str(raw if raw is not None else '')).strip()
|
||
|
|
title=vals.get('title','')
|
||
|
|
if not title:
|
||
|
|
errors.append({'row':index+1,'error':'标题为空'});continue
|
||
|
|
route=vals.get('route','')
|
||
|
|
if not route:
|
||
|
|
route=f'/{prefix or kind}/{slug_of(title)}/'
|
||
|
|
try:route=route_path(route)
|
||
|
|
except ValueError as exc:
|
||
|
|
errors.append({'row':index+1,'error':f'路径非法：{exc}'});continue
|
||
|
|
if route in seen:
|
||
|
|
errors.append({'row':index+1,'error':'导入文件内 URL 重复：'+route});continue
|
||
|
|
seen.add(route)
|
||
|
|
fields={k:v for k,v in vals.items() if k not in SYSTEM_FIELDS}
|
||
|
|
if fields.get('gallery'):
|
||
|
|
try:
|
||
|
|
values=json.loads(fields['gallery'])
|
||
|
|
if isinstance(values,list):fields['gallery']=[str(x.get('url','') if isinstance(x,dict) else x) for x in values if x]
|
||
|
|
except ValueError:
|
||
|
|
fields['gallery']=[x.strip() for x in fields['gallery'].split(';') if x.strip()][:40]
|
||
|
|
out.append({'type':kind,'title':title,'route':route,'summary':vals.get('summary',''),
|
||
|
|
'body':vals.get('body',''),'cover':vals.get('cover',''),
|
||
|
|
'seo_title':vals.get('seo_title',''),'seo_description':vals.get('seo_description',''),
|
||
|
|
'canonical':vals.get('canonical',''),'meta':{'fields':fields}})
|
||
|
|
return out,errors
|
||
|
|
|
||
|
|
|
||
|
|
def dry_run(site_id,rows,rules,kind='product',prefix=None):
|
||
|
|
if len(rows)>1000:raise ValueError('单批最多1000条内容')
|
||
|
|
items,errors=transform(rows,rules,kind,prefix)
|
||
|
|
existing={r['route'] for r in db.rows('SELECT route FROM entries WHERE site_id=?',(site_id,))}
|
||
|
|
collisions=[r['route'] for r in items if r['route'] in existing]
|
||
|
|
return {'input':len(rows),'mapped':len(items),'errors':errors,'conflicts':collisions,
|
||
|
|
'unmapped_columns':sorted(set().union(*(x.keys() for x in rows)) - set(rules.values())) if rows else [],
|
||
|
|
'sample':items[:10], 'items':items}
|
||
|
|
|
||
|
|
|
||
|
|
def commit(site_id,rows,rules,kind='product',on_conflict='skip',prefix=None):
|
||
|
|
if on_conflict not in ('skip','replace'):raise ValueError('冲突策略只能为 skip 或 replace')
|
||
|
|
report=dry_run(site_id,rows,rules,kind,prefix)
|
||
|
|
created=0;updated=0;skipped=0;errors=list(report['errors'])
|
||
|
|
for idx,item in enumerate(report['items'],1):
|
||
|
|
try:
|
||
|
|
existing=db.one('SELECT id FROM entries WHERE site_id=? AND route=?',(site_id,item['route']))
|
||
|
|
if existing and on_conflict=='skip':skipped+=1;continue
|
||
|
|
save_entry(site_id,item,existing_id=existing['id'] if existing else None,source='mapped-import')
|
||
|
|
if existing:updated+=1
|
||
|
|
else:created+=1
|
||
|
|
except Exception as exc:
|
||
|
|
errors.append({'row':idx,'error':str(exc)[:250]})
|
||
|
|
summary={'created':created,'updated':updated,'skipped':skipped,'errors':errors,'mapped':report['mapped']}
|
||
|
|
from .core import uid
|
||
|
|
db.insert('INSERT INTO import_batches VALUES (?,?,?,?,?,?)',(uid(),site_id,'',f'content-map:{kind}',json.dumps(summary,ensure_ascii=False),db.now()))
|
||
|
|
return summary
|