chore: baseline V8.10.0-dev.1 with v8_workflows type-mapping fix and trade-pro spec
- SiteForge Enterprise V8.10.0-dev.1 upstream source (210 tests) - Fix: v8_workflows._mapped_input now reads both 类型/分类 columns so WXR imports keep post/product types instead of collapsing to page - Docs: trade-pro theme port spec (R3) and implementation plan Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -0,0 +1,351 @@
|
||||
"""V8.4 bounded XML and public JSON API connectors.
|
||||
|
||||
All resulting content enters the existing staged, auditable V8 import batches.
|
||||
Public API paging runs in a subprocess under the common Worker lease/timeout.
|
||||
Authenticated API sources and database connectors are intentionally not enabled.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import ipaddress
|
||||
import re
|
||||
import uuid
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
from fastapi import APIRouter, Body, File, Form, HTTPException, Request, UploadFile
|
||||
from defusedxml import ElementTree as XML
|
||||
from defusedxml.common import DefusedXmlException
|
||||
from . import db, enterprise, jobs, safe_http, v8_workflows, secret_refs
|
||||
|
||||
router = APIRouter(prefix='/api/connectors/sites',tags=['V8.4 data connectors'])
|
||||
MAX_XML_BYTES = 8_000_000
|
||||
MAX_XML_NODES = 20_000
|
||||
MAX_XML_DEPTH = 9
|
||||
MAX_VALUE_LEN = 20_000
|
||||
MAX_RECORDS = 1000
|
||||
MAX_JSON_BYTES = 2_000_000
|
||||
TAG = re.compile(r'^[A-Za-z_][\w.-]{0,70}$')
|
||||
JSON_PATH = re.compile(r'^[A-Za-z_][\w-]{0,60}(?:\.[A-Za-z_][\w-]{0,60}){0,4}$')
|
||||
SENSITIVE_KEYS = {'key','api_key','apikey','token','access_token','secret','password','authorization','auth','signature','client_secret'}
|
||||
|
||||
|
||||
def setup():
|
||||
with db.connect() as c:
|
||||
c.executescript('''
|
||||
CREATE TABLE IF NOT EXISTS v84_api_sources (
|
||||
id TEXT PRIMARY KEY, site_id TEXT NOT NULL REFERENCES sites(id) ON DELETE CASCADE,
|
||||
url TEXT NOT NULL, items_path TEXT NOT NULL, page_param TEXT NOT NULL,
|
||||
per_page_param TEXT NOT NULL, per_page INTEGER NOT NULL,
|
||||
max_pages INTEGER NOT NULL, next_page INTEGER NOT NULL DEFAULT 1,
|
||||
scanned_pages INTEGER NOT NULL DEFAULT 0, staged_count INTEGER NOT NULL DEFAULT 0,
|
||||
kind TEXT NOT NULL, mapping_json TEXT NOT NULL, state TEXT NOT NULL DEFAULT 'ready',
|
||||
created_at TEXT NOT NULL, updated_at TEXT NOT NULL, error TEXT NOT NULL DEFAULT '');
|
||||
CREATE INDEX IF NOT EXISTS idx_v84_sources_site ON v84_api_sources(site_id,created_at);
|
||||
CREATE TABLE IF NOT EXISTS v84_api_pages (
|
||||
source_id TEXT NOT NULL REFERENCES v84_api_sources(id) ON DELETE CASCADE,
|
||||
page INTEGER NOT NULL, batch_id TEXT REFERENCES v8_import_batches(id),
|
||||
input_count INTEGER NOT NULL, created_at TEXT NOT NULL,
|
||||
PRIMARY KEY(source_id,page));
|
||||
''')
|
||||
# Preserve V8.4 sources when upgrading an existing database.
|
||||
existing={r['name'] for r in c.execute('PRAGMA table_info(v84_api_sources)')}
|
||||
for col,ddl in {
|
||||
'auth_type':"TEXT NOT NULL DEFAULT 'none'",
|
||||
'auth_header':"TEXT NOT NULL DEFAULT ''",
|
||||
'secret_ref':"TEXT NOT NULL DEFAULT ''",
|
||||
}.items():
|
||||
if col not in existing:c.execute(f'ALTER TABLE v84_api_sources ADD COLUMN {col} {ddl}')
|
||||
|
||||
|
||||
def _local_name(tag):
|
||||
return tag.rsplit('}',1)[-1] if isinstance(tag,str) else ''
|
||||
|
||||
|
||||
def parse_xml_records(data:bytes, record_tag:str):
|
||||
"""Parse a bounded, non-DTD, flat/two-level XML dataset without network access."""
|
||||
if not isinstance(data,bytes) or not 0 < len(data) <= MAX_XML_BYTES:
|
||||
raise ValueError('XML 文件大小必须在 1 字节到 8 MB 之间')
|
||||
if not isinstance(record_tag,str) or not TAG.fullmatch(record_tag):
|
||||
raise ValueError('记录节点名称不合法')
|
||||
# defusedxml is the security gate against DTD entity expansion and XXE.
|
||||
try:root=XML.fromstring(data,forbid_dtd=True)
|
||||
except (DefusedXmlException, XML.ParseError) as exc:
|
||||
raise ValueError('XML 包含不允许的实体或格式错误') from exc
|
||||
stack=[(root,1)];nodes=0;records=[]
|
||||
while stack:
|
||||
elem,depth=stack.pop();nodes+=1
|
||||
if nodes>MAX_XML_NODES or depth>MAX_XML_DEPTH:
|
||||
raise ValueError('XML 节点数或嵌套层级超限')
|
||||
if _local_name(elem.tag)==record_tag:
|
||||
records.append(elem)
|
||||
if len(records)>MAX_RECORDS:raise ValueError('XML 每次最多导入1000条记录')
|
||||
for child in reversed(list(elem)):stack.append((child,depth+1))
|
||||
if not records:raise ValueError('未找到指定的 XML 记录节点')
|
||||
rows=[]
|
||||
for record in records:
|
||||
item={}
|
||||
for child in record:
|
||||
field=_local_name(child.tag)
|
||||
if not TAG.fullmatch(field):raise ValueError('XML 字段名不合法')
|
||||
if len(child):
|
||||
for sub in child:
|
||||
if len(sub):raise ValueError('仅支持扁平或两层结构的记录')
|
||||
key=field+'.'+_local_name(sub.tag)
|
||||
if key in item:raise ValueError('XML 记录存在重复字段：'+key)
|
||||
item[key]=(sub.text or '').strip()[:MAX_VALUE_LEN]
|
||||
else:
|
||||
if field in item:raise ValueError('XML 记录存在重复字段：'+field)
|
||||
item[field]=(child.text or '').strip()[:MAX_VALUE_LEN]
|
||||
if item:rows.append(item)
|
||||
if not rows:raise ValueError('XML 记录没有可导入字段')
|
||||
if len(json.dumps(rows,ensure_ascii=False).encode('utf-8'))>v8_workflows.MAX_UPLOAD:
|
||||
raise ValueError('XML 解码后数据超过导入限制')
|
||||
return rows
|
||||
|
||||
|
||||
def _validate_url(url):
|
||||
if not isinstance(url,str) or len(url)>1600:raise ValueError('API 地址不合法')
|
||||
p=urlsplit(url)
|
||||
if p.scheme!='https' or not p.hostname or p.username or p.password or p.fragment:
|
||||
raise ValueError('API 连接器只接受公开 HTTPS 地址，不能包含账号、口令或片段')
|
||||
try:port=p.port
|
||||
except ValueError as exc:raise ValueError('API 端口不合法') from exc
|
||||
if port not in (None,443):raise ValueError('仅支持默认 HTTPS 端口')
|
||||
try:
|
||||
ip=ipaddress.ip_address(p.hostname)
|
||||
if not ip.is_global:raise ValueError('禁止使用非公网 IP API 地址')
|
||||
except ValueError as exc:
|
||||
if str(exc)=='禁止使用非公网 IP API 地址':raise
|
||||
if any(k.casefold() in SENSITIVE_KEYS or 'token' in k.casefold() or 'secret' in k.casefold()
|
||||
for k,_ in parse_qsl(p.query,keep_blank_values=True)):
|
||||
raise ValueError('请勿在 API URL 中传入凭据或令牌')
|
||||
if len(parse_qsl(p.query,keep_blank_values=True))>30:raise ValueError('URL 查询参数过多')
|
||||
# DNS->socket pinning and non-public IP protection are enforced on fetch.
|
||||
return urlunsplit(p)
|
||||
|
||||
|
||||
def create_api_source(site_id,payload):
|
||||
v8_workflows._require_site(site_id)
|
||||
if not isinstance(payload,dict):raise ValueError('API 参数必须是对象')
|
||||
url=_validate_url(payload.get('url'))
|
||||
kind=str(payload.get('content_type','product'))
|
||||
if kind not in ('product','vehicle','page','post','brand','case','service','other'):
|
||||
raise ValueError('内容类型不受支持')
|
||||
items_path=str(payload.get('items_path','items'))
|
||||
if not JSON_PATH.fullmatch(items_path):raise ValueError('JSON 路径不合法')
|
||||
page_param=str(payload.get('page_param','page'))
|
||||
per_page_param=str(payload.get('per_page_param','per_page'))
|
||||
if not TAG.fullmatch(page_param) or not TAG.fullmatch(per_page_param) or page_param==per_page_param:
|
||||
raise ValueError('分页参数不合法')
|
||||
if any(k in (page_param,per_page_param) for k,_ in parse_qsl(urlsplit(url).query)):
|
||||
raise ValueError('源 URL 不应携带分页参数')
|
||||
try:page_size=int(payload.get('page_size',50));max_pages=int(payload.get('max_pages',20))
|
||||
except (TypeError,ValueError):raise ValueError('分页数量不合法')
|
||||
if not 1<=page_size<=100 or not 1<=max_pages<=50 or page_size*max_pages>5000:
|
||||
raise ValueError('上限为每页100条、最多50页、合计5000条')
|
||||
mapping=payload.get('mapping') or {}
|
||||
if not isinstance(mapping,dict) or len(mapping)>100 or any(not isinstance(k,str) or not isinstance(v,str) or len(v)>100 for k,v in mapping.items()):
|
||||
raise ValueError('字段映射规则不合法')
|
||||
auth_type=str(payload.get('auth_type','none'))
|
||||
if auth_type not in ('none','bearer','api_key','custom'):
|
||||
raise ValueError('不支持的 API 认证方式')
|
||||
secret_ref='';auth_header=''
|
||||
if auth_type!='none':
|
||||
secret_ref=secret_refs.validate_reference(payload.get('secret_ref'))
|
||||
auth_header='Authorization' if auth_type=='bearer' else 'X-API-Key'
|
||||
if auth_type=='custom':
|
||||
auth_header=str(payload.get('auth_header',''))
|
||||
if not re.fullmatch(r'X-[A-Za-z0-9-]{2,64}',auth_header,re.I):
|
||||
raise ValueError('自定义凭据请求头必须以 X- 开头')
|
||||
elif payload.get('secret_ref'):
|
||||
raise ValueError('公开 API 不应保存密钥引用')
|
||||
source_id=uuid.uuid4().hex
|
||||
db.insert('''INSERT INTO v84_api_sources(id,site_id,url,items_path,page_param,per_page_param,
|
||||
per_page,max_pages,kind,mapping_json,auth_type,auth_header,secret_ref,created_at,updated_at)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''',
|
||||
(source_id,site_id,url,items_path,page_param,per_page_param,page_size,max_pages,kind,
|
||||
json.dumps(mapping,ensure_ascii=False),auth_type,auth_header,secret_ref,db.now(),db.now()))
|
||||
return status(site_id,source_id)
|
||||
|
||||
|
||||
def _source(site_id,source_id):
|
||||
r=db.one('SELECT * FROM v84_api_sources WHERE id=? AND site_id=?',(source_id,site_id))
|
||||
if not r:raise HTTPException(404,'API 数据源不存在')
|
||||
return r
|
||||
|
||||
|
||||
def status(site_id,source_id):
|
||||
s=_source(site_id,source_id)
|
||||
# Secret references are never returned by management APIs.
|
||||
return {k:s[k] for k in ('id','site_id','url','items_path','state','next_page',
|
||||
'scanned_pages','staged_count','max_pages','per_page','kind','error','auth_type','auth_header')} | {
|
||||
'batches':db.rows('SELECT page,batch_id,input_count FROM v84_api_pages WHERE source_id=? ORDER BY page',(source_id,))}
|
||||
|
||||
|
||||
def _data_items(content,items_path):
|
||||
try:result=json.loads(content.decode('utf-8-sig'))
|
||||
except (ValueError,UnicodeError) as exc:raise ValueError('API 响应不是有效 UTF-8 JSON') from exc
|
||||
if isinstance(result,list):items=result
|
||||
else:
|
||||
for part in items_path.split('.'):
|
||||
if not isinstance(result,dict) or part not in result:raise ValueError('API JSON 路径不存在：'+items_path)
|
||||
result=result[part]
|
||||
items=result
|
||||
if not isinstance(items,list) or len(items)>100 or any(not isinstance(x,dict) for x in items):
|
||||
raise ValueError('API 当前页必须包含最多100条对象记录')
|
||||
if any(len(row)>150 for row in items):raise ValueError('API 单条字段数量超过限制')
|
||||
return items
|
||||
|
||||
|
||||
def page_url(source,page):
|
||||
p=urlsplit(source['url']);params=parse_qsl(p.query,keep_blank_values=True)
|
||||
params.extend(((source['page_param'],str(page)),(source['per_page_param'],str(source['per_page']))))
|
||||
return urlunsplit((p.scheme,p.netloc,p.path,urlencode(params),''))
|
||||
|
||||
|
||||
def scan_page(site_id,source_id):
|
||||
source=_source(site_id,source_id)
|
||||
if source['state'] in ('cancelled','completed','limit_reached'):return {'state':source['state'],'next_page':source['next_page']}
|
||||
if source['next_page']>source['max_pages']:
|
||||
db.execute("UPDATE v84_api_sources SET state='completed',updated_at=? WHERE id=?",(db.now(),source_id))
|
||||
return {'state':'completed','next_page':source['next_page']}
|
||||
page=int(source['next_page'])
|
||||
# A process crash after stage but before the checkpoint must not stage twice.
|
||||
existing=db.one('SELECT * FROM v84_api_pages WHERE source_id=? AND page=?',(source_id,page))
|
||||
origin=f'api:{source_id}:page:{page}'
|
||||
if not existing:
|
||||
pending=db.one('SELECT id,input_count FROM v8_import_batches WHERE site_id=? AND origin=?',(site_id,origin))
|
||||
if pending:
|
||||
batch_id=pending['id'];count=pending['input_count']
|
||||
else:
|
||||
extra={}
|
||||
if source.get('auth_type','none')!='none':
|
||||
token=secret_refs.resolve(source['secret_ref'])
|
||||
extra[source['auth_header']]=('Bearer '+token if source['auth_type']=='bearer' else token)
|
||||
opts={'headers':{'Accept':'application/json'},'redirects':0,'timeout':20,'max_bytes':MAX_JSON_BYTES}
|
||||
if extra:opts['credential_headers']=extra
|
||||
resp=safe_http.fetch(page_url(source,page),**opts)
|
||||
resp.raise_for_status()
|
||||
mime=resp.headers.get('content-type','').split(';')[0].strip().lower()
|
||||
if mime!='application/json' and not mime.endswith('+json'):
|
||||
raise ValueError('API 响应 Content-Type 必须为 JSON')
|
||||
rows=_data_items(resp.content,source['items_path'])
|
||||
count=len(rows);batch_id=None
|
||||
if rows:
|
||||
mapping=json.loads(source['mapping_json'] or '{}') or None
|
||||
keys=[f'{source_id}:{page}:{i}:{str(r.get("id",i))[:60]}' for i,r in enumerate(rows)]
|
||||
staged=v8_workflows.stage(site_id,rows,mapping,source['kind'],origin,keys)
|
||||
batch_id=staged['batch_id']
|
||||
with db.connect() as c:
|
||||
c.execute('BEGIN IMMEDIATE')
|
||||
# The exclusive one-page checkpoint is idempotent across retries.
|
||||
c.execute('''INSERT INTO v84_api_pages(source_id,page,batch_id,input_count,created_at)
|
||||
VALUES(?,?,?,?,?) ON CONFLICT(source_id,page) DO NOTHING''',
|
||||
(source_id,page,batch_id,count,db.now()))
|
||||
else:
|
||||
count=existing['input_count'];batch_id=existing['batch_id']
|
||||
finished=count<source['per_page'] or page>=source['max_pages']
|
||||
final_state=('limit_reached' if page>=source['max_pages'] and count>=source['per_page'] else 'completed') if finished else 'ready'
|
||||
with db.connect() as c:
|
||||
c.execute('BEGIN IMMEDIATE')
|
||||
cur=c.execute('SELECT next_page,state FROM v84_api_sources WHERE id=? AND site_id=?',(source_id,site_id)).fetchone()
|
||||
if cur and cur['state']=='cancelled':
|
||||
return {'state':'cancelled','page':page,'count':count,'batch_id':batch_id,'next_page':page}
|
||||
if cur and cur['next_page']==page:
|
||||
c.execute('''UPDATE v84_api_sources SET next_page=?,scanned_pages=scanned_pages+1,
|
||||
staged_count=staged_count+?,state=?,updated_at=?,error='' WHERE id=?''',
|
||||
(page+1,count,final_state,db.now(),source_id))
|
||||
return {'state':final_state,'page':page,'count':count,'batch_id':batch_id,
|
||||
'next_page':page+1}
|
||||
|
||||
|
||||
def run_scan(site_id,source_id,job_id=None):
|
||||
source=_source(site_id,source_id)
|
||||
if source['state']=='cancelled':return status(site_id,source_id)
|
||||
# Worker retry always resumes from the durable checkpoint.
|
||||
while True:
|
||||
one=scan_page(site_id,source_id)
|
||||
if one['state'] in ('completed','limit_reached'):break
|
||||
if one['state']=='cancelled':break
|
||||
return status(site_id,source_id)
|
||||
|
||||
|
||||
def submit_scan(site_id,source_id,actor,key=None):
|
||||
source=_source(site_id,source_id)
|
||||
if source['state']=='cancelled':raise HTTPException(409,'数据源已取消')
|
||||
if key is not None and not (8<=len(key)<=128 and all(c.isalnum() or c in '_-:.' for c in key)):
|
||||
raise ValueError('幂等键不合法')
|
||||
payload=json.dumps({'source_id':source_id},sort_keys=True,separators=(',',':'))
|
||||
with db.connect() as c:
|
||||
c.execute('BEGIN IMMEDIATE')
|
||||
if key:
|
||||
prior=c.execute('SELECT id,state,kind,payload_json FROM build_jobs WHERE site_id=? AND idempotency_key=?',(site_id,key)).fetchone()
|
||||
if prior:
|
||||
if prior['kind']!='api_scan' or prior['payload_json']!=payload:raise ValueError('幂等键已关联其他任务')
|
||||
return {'id':prior['id'],'state':prior['state'],'existing':True}
|
||||
prior=c.execute('''SELECT id,state FROM build_jobs WHERE site_id=? AND kind='api_scan'
|
||||
AND state IN ('queued','running','retrying') AND payload_json=? LIMIT 1''',(site_id,payload)).fetchone()
|
||||
if prior:return {'id':prior['id'],'state':prior['state'],'existing':True}
|
||||
if source['state'] in ('completed','limit_reached'):return {'state':source['state'],'existing':True,'source_id':source_id}
|
||||
if c.execute("SELECT COUNT(*) FROM build_jobs WHERE state IN ('queued','running','retrying')").fetchone()[0]>=jobs._limit('SITEFORGE_MAX_PENDING_JOBS',100):
|
||||
raise jobs.JobCapacityError('全局任务容量已满')
|
||||
if c.execute("SELECT COUNT(*) FROM build_jobs WHERE site_id=? AND state IN ('queued','running','retrying')",(site_id,)).fetchone()[0]>=jobs._limit('SITEFORGE_MAX_PENDING_JOBS_PER_SITE',5):
|
||||
raise jobs.JobCapacityError('当前网站任务容量已满')
|
||||
jid=uuid.uuid4().hex[:16]
|
||||
c.execute('''INSERT INTO build_jobs(id,site_id,kind,state,created_by,created_at,
|
||||
payload_json,idempotency_key,max_attempts,timeout_seconds,available_at)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?)''',
|
||||
(jid,site_id,'api_scan','queued',actor,db.now(),payload,key,3,900,db.now()))
|
||||
from . import redis_signal
|
||||
redis_signal.notify()
|
||||
return {'id':jid,'state':'queued','existing':False}
|
||||
|
||||
|
||||
@router.post('/{site_id}/xml/staging')
|
||||
async def api_xml(request:Request,site_id:str,file:UploadFile=File(...),record_tag:str=Form('item'),
|
||||
content_type:str=Form('product'),mapping_json:str=Form('{}')):
|
||||
enterprise.require(request,site_id,'editor')
|
||||
blob=await file.read(MAX_XML_BYTES+1)
|
||||
try:
|
||||
mapping=json.loads(mapping_json)
|
||||
rows=parse_xml_records(blob,record_tag)
|
||||
result=v8_workflows.stage(site_id,rows,mapping or None,content_type,'xml:'+str(file.filename or 'upload.xml')[:100])
|
||||
return {**result,'columns':list(rows[0]),'record_tag':record_tag}
|
||||
except (ValueError,TypeError,json.JSONDecodeError) as exc:raise HTTPException(400,str(exc))
|
||||
|
||||
|
||||
@router.post('/{site_id}/api-sources')
|
||||
def api_create(request:Request,site_id:str,payload:dict=Body(...)):
|
||||
enterprise.require(request,site_id,'editor')
|
||||
try:return create_api_source(site_id,payload)
|
||||
except ValueError as exc:raise HTTPException(400,str(exc))
|
||||
|
||||
|
||||
@router.get('/{site_id}/api-sources')
|
||||
def api_list(request:Request,site_id:str):
|
||||
enterprise.require(request,site_id,'viewer')
|
||||
v8_workflows._require_site(site_id)
|
||||
return {'items':db.rows('SELECT id,url,state,next_page,staged_count FROM v84_api_sources WHERE site_id=? ORDER BY created_at DESC LIMIT 100',(site_id,))}
|
||||
|
||||
|
||||
@router.get('/{site_id}/api-sources/{source_id}')
|
||||
def api_status(request:Request,site_id:str,source_id:str):
|
||||
enterprise.require(request,site_id,'viewer')
|
||||
return status(site_id,source_id)
|
||||
|
||||
|
||||
@router.post('/{site_id}/api-sources/{source_id}/scan',status_code=202)
|
||||
def api_scan(request:Request,site_id:str,source_id:str,payload:dict=Body(default_factory=dict)):
|
||||
enterprise.require(request,site_id,'editor')
|
||||
try:return submit_scan(site_id,source_id,request.state.actor,payload.get('idempotency_key'))
|
||||
except (ValueError,jobs.JobCapacityError) as exc:raise HTTPException(409,str(exc))
|
||||
|
||||
|
||||
@router.post('/{site_id}/api-sources/{source_id}/cancel')
|
||||
def api_cancel(request:Request,site_id:str,source_id:str):
|
||||
enterprise.require(request,site_id,'editor')
|
||||
s=_source(site_id,source_id)
|
||||
if s['state']!='completed':db.execute("UPDATE v84_api_sources SET state='cancelled',updated_at=? WHERE id=?",(db.now(),source_id))
|
||||
for row in db.rows("SELECT id FROM build_jobs WHERE site_id=? AND kind='api_scan' AND state IN ('queued','running','retrying')",(site_id,)):
|
||||
job=db.one('SELECT payload_json FROM build_jobs WHERE id=?',(row['id'],))
|
||||
if job and json.loads(job['payload_json']).get('source_id')==source_id:jobs.cancel(site_id,row['id'])
|
||||
return status(site_id,source_id)
|
||||
Reference in New Issue
Block a user