cms-SiteForge/app/assembly.py

"""Declarative mapping from arbitrary tabular source data to SiteForge content.
Never eval mappings, no AI hallucinated fields, preview shows all errors.
"""
import json,re,hashlib
from . import db
from .core import route_path,slug_of,save_entry

ALIASES={
 'title':('title','name','product_name','model_name','名称','标题','产品名称','车型','车型名称','商品名称'),
 'route':('route','url','path','permalink','链接','路径','旧站链接','原网址'),
 'summary':('summary','description','excerpt','简介','摘要','产品简介'),
 'body':('body','content','html','正文','内容','详细介绍'),
 'cover':('cover','image','thumbnail','featured_image','图片','封面','主图'),
 'brand':('brand','manufacturer','品牌','厂商'),
 'category':('category','分类','类目'),
 'seo_title':('seo_title','meta_title','SEO标题','seo标题'),
 'seo_description':('seo_description','meta_description','SEO描述','seo描述'),
 'canonical':('canonical','规范网址'),
 'model':('model','model_name','车型型号','型号','规格型号'),
 'energy':('energy','energy_type','fuel_type','动力形式','能源类型','燃料类型'),
 'range':('range','battery_range','纯电续航','续航','续航里程'),
 'year':('year','model_year','年款','年份'),
 'sku':('sku','product_sku','产品编号','货号'),
 'gallery':('gallery','images','image_gallery','产品图片','图库','详情图片'),
 'price':('price','msrp','售价','价格'),
 'specs':('specs','attributes','parameters','技术参数','产品参数'),

}
SYSTEM_FIELDS={'title','route','summary','body','cover','seo_title','seo_description','canonical'}


def setup():
    with db.connect() as c:
        c.executescript('''CREATE TABLE IF NOT EXISTS mapping_profiles (
          id TEXT PRIMARY KEY,site_id TEXT NOT NULL REFERENCES sites(id) ON DELETE CASCADE,
          name TEXT NOT NULL,content_type TEXT NOT NULL, rules_json TEXT NOT NULL,
          created_at TEXT NOT NULL,updated_at TEXT NOT NULL);
          CREATE TABLE IF NOT EXISTS import_batches (
          id TEXT PRIMARY KEY,site_id TEXT NOT NULL REFERENCES sites(id) ON DELETE CASCADE,
          profile_id TEXT, source TEXT NOT NULL, result_json TEXT NOT NULL,
          created_at TEXT NOT NULL);
        ''')


def suggest(columns):
    clean=lambda x:re.sub(r'[\s\-_]+','',str(x).casefold())
    src={clean(col):col for col in columns}
    mapping={}
    for target,aliases in ALIASES.items():
        for alias in aliases:
            if clean(alias) in src:
                mapping[target]=src[clean(alias)];break
    return mapping


def source_column(spec):
    return spec if isinstance(spec,str) else spec['source']


def validate_rules(rules,columns):
    if not isinstance(rules,dict) or len(rules)>100:raise ValueError('字段映射规则不合法')
    for dest,spec in rules.items():
        if not isinstance(dest,str) or not re.fullmatch(r'[\w\-]{1,60}',dest,re.UNICODE):
            raise ValueError('目标字段名称不合法')
        if isinstance(spec,dict):
            allowed={'source','trim','case','default','prefix','suffix'}
            if set(spec)-allowed or 'source' not in spec:raise ValueError('包含不支持的转换配置')
            if type(spec.get('trim',True)) is not bool:raise ValueError('trim 必须为布尔值')
            if spec.get('case','none') not in ('none','upper','lower'):raise ValueError('不支持的大小写处理')
            for key in ('default','prefix','suffix'):
                if key in spec and (not isinstance(spec[key],str) or len(spec[key])>500):
                    raise ValueError('转换字符串过长或格式非法')
        elif not isinstance(spec,str):raise ValueError('映射字段格式非法')
        src=source_column(spec)
        if not isinstance(src,str) or src not in columns:raise ValueError(f'来源字段不存在: {src}')
    if 'title' not in rules:raise ValueError('必须映射标题字段')


def mapped_value(row,spec):
    raw=row.get(source_column(spec))
    value=json.dumps(raw,ensure_ascii=False) if isinstance(raw,(dict,list)) else str(raw if raw is not None else '')
    if isinstance(spec,str):return value.strip()
    if spec.get('trim',True):value=value.strip()
    if not value:value=spec.get('default','')
    case=spec.get('case','none')
    if case=='upper':value=value.upper()
    elif case=='lower':value=value.lower()
    return spec.get('prefix','')+value+spec.get('suffix','')


def transform(rows,rules,kind='product',prefix=None):
    if kind not in ('product','post','page','brand','vehicle','service','case','other'):raise ValueError('不支持的内容类型')
    columns=set().union(*(r.keys() for r in rows)) if rows else set()
    validate_rules(rules,columns)
    seen=set();out=[];errors=[]
    for index,row in enumerate(rows):
        vals={}
        for target,spec in rules.items():
            vals[target]=mapped_value(row,spec)
        title=vals.get('title','')
        if not title:
            errors.append({'row':index+1,'error':'标题为空'});continue
        route=vals.get('route','')
        if not route:
            route=f'/{prefix or kind}/{slug_of(title)}/'
        try:route=route_path(route)
        except ValueError as exc:
            errors.append({'row':index+1,'error':f'路径非法：{exc}'});continue
        if route in seen:
            errors.append({'row':index+1,'error':'导入文件内 URL 重复：'+route});continue
        seen.add(route)
        fields={k:v for k,v in vals.items() if k not in SYSTEM_FIELDS}
        if fields.get('gallery'):
            try:
                values=json.loads(fields['gallery'])
                if isinstance(values,list):fields['gallery']=[str(x.get('url','') if isinstance(x,dict) else x) for x in values if x]
            except ValueError:
                fields['gallery']=[x.strip() for x in fields['gallery'].split(';') if x.strip()][:40]
        out.append({'type':kind,'title':title,'route':route,'summary':vals.get('summary',''),
                    'body':vals.get('body',''),'cover':vals.get('cover',''),
                    'seo_title':vals.get('seo_title',''),'seo_description':vals.get('seo_description',''),
                    'canonical':vals.get('canonical',''),'meta':{'fields':fields}})
    return out,errors


def _state_token(site_id,items):
    """Preview token binds mapped payload and existing rows' revision metadata.

    This is advisory optimistic checking, not a global distributed lock.
    """
    routes=[x['route'] for x in items]
    existing={r['route']:r for r in
              db.rows('SELECT * FROM entries WHERE site_id=?',(site_id,))
              if r['route'] in routes}
    state={'site':site_id,'items':items,'existing':existing}
    return hashlib.sha256(json.dumps(state,ensure_ascii=False,sort_keys=True,
                                    separators=(',',':')).encode('utf-8')).hexdigest()


def dry_run(site_id,rows,rules,kind='product',prefix=None):
    if len(rows)>1000:raise ValueError('单批最多1000条内容')
    items,errors=transform(rows,rules,kind,prefix)
    existing={r['route'] for r in db.rows('SELECT route FROM entries WHERE site_id=?',(site_id,))}
    collisions=[r['route'] for r in items if r['route'] in existing]
    return {'input':len(rows),'mapped':len(items),'errors':errors,'conflicts':collisions,
            'unmapped_columns':sorted(set().union(*(x.keys() for x in rows)) -
                                       {source_column(v) for v in rules.values()}) if rows else [],
            'sample':items[:10], 'items':items,
            'plan_hash':_state_token(site_id,items),
            'supported_transforms':['trim','default','upper','lower','prefix','suffix'],
            'note':'提交时携带 expected_plan_hash 可阻止预检后发生变化的数据覆盖'}


def commit(site_id,rows,rules,kind='product',on_conflict='skip',prefix=None,expected_plan_hash=None):
    if on_conflict not in ('skip','replace','abort'):
        raise ValueError('冲突策略只能为 skip、replace 或 abort')
    report=dry_run(site_id,rows,rules,kind,prefix)
    if expected_plan_hash is not None:
        if not isinstance(expected_plan_hash,str) or not re.fullmatch(r'[a-f0-9]{64}',expected_plan_hash):
            raise ValueError('预检令牌格式无效')
        if expected_plan_hash!=report['plan_hash']:
            raise ValueError('导入预检已过期：来源数据或已有内容发生变化，请重新预览')
    if on_conflict=='abort' and (report['conflicts'] or report['errors']):
        raise ValueError('严格导入被阻断：存在 URL 冲突或无效数据，请处理后重新预览')
    created=0;updated=0;skipped=0;errors=list(report['errors'])
    for idx,item in enumerate(report['items'],1):
        try:
            existing=db.one('SELECT id FROM entries WHERE site_id=? AND route=?',(site_id,item['route']))
            if existing and on_conflict=='skip':skipped+=1;continue
            # abort semantics require a preflight without errors; do not turn
            # this into an automatic 'replace' policy on a colliding route.
            if existing and on_conflict=='abort':raise ValueError('提交期间出现 URL 冲突')
            save_entry(site_id,item,existing_id=existing['id'] if existing else None,source='mapped-import')
            if existing:updated+=1
            else:created+=1
        except Exception as exc:
            errors.append({'row':idx,'error':str(exc)[:250]})
            if on_conflict=='abort':break
    summary={'created':created,'updated':updated,'skipped':skipped,'errors':errors,'mapped':report['mapped'],
             'plan_hash':report['plan_hash']}
    from .core import uid
    db.insert('INSERT INTO import_batches VALUES (?,?,?,?,?,?)',(uid(),site_id,'',f'content-map:{kind}',json.dumps(summary,ensure_ascii=False),db.now()))
    return summary