"""V8.4 bounded XML and public JSON API connectors. All resulting content enters the existing staged, auditable V8 import batches. Public API paging runs in a subprocess under the common Worker lease/timeout. Authenticated API sources and database connectors are intentionally not enabled. """ from __future__ import annotations import json import ipaddress import re import uuid from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit from fastapi import APIRouter, Body, File, Form, HTTPException, Request, UploadFile from defusedxml import ElementTree as XML from defusedxml.common import DefusedXmlException from . import db, enterprise, jobs, safe_http, v8_workflows, secret_refs router = APIRouter(prefix='/api/connectors/sites',tags=['V8.4 data connectors']) MAX_XML_BYTES = 8_000_000 MAX_XML_NODES = 20_000 MAX_XML_DEPTH = 9 MAX_VALUE_LEN = 20_000 MAX_RECORDS = 1000 MAX_JSON_BYTES = 2_000_000 TAG = re.compile(r'^[A-Za-z_][\w.-]{0,70}$') JSON_PATH = re.compile(r'^[A-Za-z_][\w-]{0,60}(?:\.[A-Za-z_][\w-]{0,60}){0,4}$') SENSITIVE_KEYS = {'key','api_key','apikey','token','access_token','secret','password','authorization','auth','signature','client_secret'} def setup(): with db.connect() as c: c.executescript(''' CREATE TABLE IF NOT EXISTS v84_api_sources ( id TEXT PRIMARY KEY, site_id TEXT NOT NULL REFERENCES sites(id) ON DELETE CASCADE, url TEXT NOT NULL, items_path TEXT NOT NULL, page_param TEXT NOT NULL, per_page_param TEXT NOT NULL, per_page INTEGER NOT NULL, max_pages INTEGER NOT NULL, next_page INTEGER NOT NULL DEFAULT 1, scanned_pages INTEGER NOT NULL DEFAULT 0, staged_count INTEGER NOT NULL DEFAULT 0, kind TEXT NOT NULL, mapping_json TEXT NOT NULL, state TEXT NOT NULL DEFAULT 'ready', created_at TEXT NOT NULL, updated_at TEXT NOT NULL, error TEXT NOT NULL DEFAULT ''); CREATE INDEX IF NOT EXISTS idx_v84_sources_site ON v84_api_sources(site_id,created_at); CREATE TABLE IF NOT EXISTS v84_api_pages ( source_id TEXT NOT NULL REFERENCES v84_api_sources(id) ON DELETE CASCADE, page INTEGER NOT NULL, batch_id TEXT REFERENCES v8_import_batches(id), input_count INTEGER NOT NULL, created_at TEXT NOT NULL, PRIMARY KEY(source_id,page)); ''') # Preserve V8.4 sources when upgrading an existing database. existing={r['name'] for r in c.execute('PRAGMA table_info(v84_api_sources)')} for col,ddl in { 'auth_type':"TEXT NOT NULL DEFAULT 'none'", 'auth_header':"TEXT NOT NULL DEFAULT ''", 'secret_ref':"TEXT NOT NULL DEFAULT ''", }.items(): if col not in existing:c.execute(f'ALTER TABLE v84_api_sources ADD COLUMN {col} {ddl}') def _local_name(tag): return tag.rsplit('}',1)[-1] if isinstance(tag,str) else '' def parse_xml_records(data:bytes, record_tag:str): """Parse a bounded, non-DTD, flat/two-level XML dataset without network access.""" if not isinstance(data,bytes) or not 0 < len(data) <= MAX_XML_BYTES: raise ValueError('XML 文件大小必须在 1 字节到 8 MB 之间') if not isinstance(record_tag,str) or not TAG.fullmatch(record_tag): raise ValueError('记录节点名称不合法') # defusedxml is the security gate against DTD entity expansion and XXE. try:root=XML.fromstring(data,forbid_dtd=True) except (DefusedXmlException, XML.ParseError) as exc: raise ValueError('XML 包含不允许的实体或格式错误') from exc stack=[(root,1)];nodes=0;records=[] while stack: elem,depth=stack.pop();nodes+=1 if nodes>MAX_XML_NODES or depth>MAX_XML_DEPTH: raise ValueError('XML 节点数或嵌套层级超限') if _local_name(elem.tag)==record_tag: records.append(elem) if len(records)>MAX_RECORDS:raise ValueError('XML 每次最多导入1000条记录') for child in reversed(list(elem)):stack.append((child,depth+1)) if not records:raise ValueError('未找到指定的 XML 记录节点') rows=[] for record in records: item={} for child in record: field=_local_name(child.tag) if not TAG.fullmatch(field):raise ValueError('XML 字段名不合法') if len(child): for sub in child: if len(sub):raise ValueError('仅支持扁平或两层结构的记录') key=field+'.'+_local_name(sub.tag) if key in item:raise ValueError('XML 记录存在重复字段：'+key) item[key]=(sub.text or '').strip()[:MAX_VALUE_LEN] else: if field in item:raise ValueError('XML 记录存在重复字段：'+field) item[field]=(child.text or '').strip()[:MAX_VALUE_LEN] if item:rows.append(item) if not rows:raise ValueError('XML 记录没有可导入字段') if len(json.dumps(rows,ensure_ascii=False).encode('utf-8'))>v8_workflows.MAX_UPLOAD: raise ValueError('XML 解码后数据超过导入限制') return rows def _validate_url(url): if not isinstance(url,str) or len(url)>1600:raise ValueError('API 地址不合法') p=urlsplit(url) if p.scheme!='https' or not p.hostname or p.username or p.password or p.fragment: raise ValueError('API 连接器只接受公开 HTTPS 地址，不能包含账号、口令或片段') try:port=p.port except ValueError as exc:raise ValueError('API 端口不合法') from exc if port not in (None,443):raise ValueError('仅支持默认 HTTPS 端口') try: ip=ipaddress.ip_address(p.hostname) if not ip.is_global:raise ValueError('禁止使用非公网 IP API 地址') except ValueError as exc: if str(exc)=='禁止使用非公网 IP API 地址':raise if any(k.casefold() in SENSITIVE_KEYS or 'token' in k.casefold() or 'secret' in k.casefold() for k,_ in parse_qsl(p.query,keep_blank_values=True)): raise ValueError('请勿在 API URL 中传入凭据或令牌') if len(parse_qsl(p.query,keep_blank_values=True))>30:raise ValueError('URL 查询参数过多') # DNS->socket pinning and non-public IP protection are enforced on fetch. return urlunsplit(p) def create_api_source(site_id,payload): v8_workflows._require_site(site_id) if not isinstance(payload,dict):raise ValueError('API 参数必须是对象') url=_validate_url(payload.get('url')) kind=str(payload.get('content_type','product')) if kind not in ('product','vehicle','page','post','brand','case','service','other'): raise ValueError('内容类型不受支持') items_path=str(payload.get('items_path','items')) if not JSON_PATH.fullmatch(items_path):raise ValueError('JSON 路径不合法') page_param=str(payload.get('page_param','page')) per_page_param=str(payload.get('per_page_param','per_page')) if not TAG.fullmatch(page_param) or not TAG.fullmatch(per_page_param) or page_param==per_page_param: raise ValueError('分页参数不合法') if any(k in (page_param,per_page_param) for k,_ in parse_qsl(urlsplit(url).query)): raise ValueError('源 URL 不应携带分页参数') try:page_size=int(payload.get('page_size',50));max_pages=int(payload.get('max_pages',20)) except (TypeError,ValueError):raise ValueError('分页数量不合法') if not 1<=page_size<=100 or not 1<=max_pages<=50 or page_size*max_pages>5000: raise ValueError('上限为每页100条、最多50页、合计5000条') mapping=payload.get('mapping') or {} if not isinstance(mapping,dict) or len(mapping)>100 or any(not isinstance(k,str) or not isinstance(v,str) or len(v)>100 for k,v in mapping.items()): raise ValueError('字段映射规则不合法') auth_type=str(payload.get('auth_type','none')) if auth_type not in ('none','bearer','api_key','custom'): raise ValueError('不支持的 API 认证方式') secret_ref='';auth_header='' if auth_type!='none': secret_ref=secret_refs.validate_reference(payload.get('secret_ref')) auth_header='Authorization' if auth_type=='bearer' else 'X-API-Key' if auth_type=='custom': auth_header=str(payload.get('auth_header','')) if not re.fullmatch(r'X-[A-Za-z0-9-]{2,64}',auth_header,re.I): raise ValueError('自定义凭据请求头必须以 X- 开头') elif payload.get('secret_ref'): raise ValueError('公开 API 不应保存密钥引用') source_id=uuid.uuid4().hex db.insert('''INSERT INTO v84_api_sources(id,site_id,url,items_path,page_param,per_page_param, per_page,max_pages,kind,mapping_json,auth_type,auth_header,secret_ref,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', (source_id,site_id,url,items_path,page_param,per_page_param,page_size,max_pages,kind, json.dumps(mapping,ensure_ascii=False),auth_type,auth_header,secret_ref,db.now(),db.now())) return status(site_id,source_id) def _source(site_id,source_id): r=db.one('SELECT * FROM v84_api_sources WHERE id=? AND site_id=?',(source_id,site_id)) if not r:raise HTTPException(404,'API 数据源不存在') return r def status(site_id,source_id): s=_source(site_id,source_id) # Secret references are never returned by management APIs. return {k:s[k] for k in ('id','site_id','url','items_path','state','next_page', 'scanned_pages','staged_count','max_pages','per_page','kind','error','auth_type','auth_header')} | { 'batches':db.rows('SELECT page,batch_id,input_count FROM v84_api_pages WHERE source_id=? ORDER BY page',(source_id,))} def _data_items(content,items_path): try:result=json.loads(content.decode('utf-8-sig')) except (ValueError,UnicodeError) as exc:raise ValueError('API 响应不是有效 UTF-8 JSON') from exc if isinstance(result,list):items=result else: for part in items_path.split('.'): if not isinstance(result,dict) or part not in result:raise ValueError('API JSON 路径不存在：'+items_path) result=result[part] items=result if not isinstance(items,list) or len(items)>100 or any(not isinstance(x,dict) for x in items): raise ValueError('API 当前页必须包含最多100条对象记录') if any(len(row)>150 for row in items):raise ValueError('API 单条字段数量超过限制') return items def page_url(source,page): p=urlsplit(source['url']);params=parse_qsl(p.query,keep_blank_values=True) params.extend(((source['page_param'],str(page)),(source['per_page_param'],str(source['per_page'])))) return urlunsplit((p.scheme,p.netloc,p.path,urlencode(params),'')) def scan_page(site_id,source_id): source=_source(site_id,source_id) if source['state'] in ('cancelled','completed','limit_reached'):return {'state':source['state'],'next_page':source['next_page']} if source['next_page']>source['max_pages']: db.execute("UPDATE v84_api_sources SET state='completed',updated_at=? WHERE id=?",(db.now(),source_id)) return {'state':'completed','next_page':source['next_page']} page=int(source['next_page']) # A process crash after stage but before the checkpoint must not stage twice. existing=db.one('SELECT * FROM v84_api_pages WHERE source_id=? AND page=?',(source_id,page)) origin=f'api:{source_id}:page:{page}' if not existing: pending=db.one('SELECT id,input_count FROM v8_import_batches WHERE site_id=? AND origin=?',(site_id,origin)) if pending: batch_id=pending['id'];count=pending['input_count'] else: extra={} if source.get('auth_type','none')!='none': token=secret_refs.resolve(source['secret_ref']) extra[source['auth_header']]=('Bearer '+token if source['auth_type']=='bearer' else token) opts={'headers':{'Accept':'application/json'},'redirects':0,'timeout':20,'max_bytes':MAX_JSON_BYTES} if extra:opts['credential_headers']=extra resp=safe_http.fetch(page_url(source,page),**opts) resp.raise_for_status() mime=resp.headers.get('content-type','').split(';')[0].strip().lower() if mime!='application/json' and not mime.endswith('+json'): raise ValueError('API 响应 Content-Type 必须为 JSON') rows=_data_items(resp.content,source['items_path']) count=len(rows);batch_id=None if rows: mapping=json.loads(source['mapping_json'] or '{}') or None keys=[f'{source_id}:{page}:{i}:{str(r.get("id",i))[:60]}' for i,r in enumerate(rows)] staged=v8_workflows.stage(site_id,rows,mapping,source['kind'],origin,keys) batch_id=staged['batch_id'] with db.connect() as c: c.execute('BEGIN IMMEDIATE') # The exclusive one-page checkpoint is idempotent across retries. c.execute('''INSERT INTO v84_api_pages(source_id,page,batch_id,input_count,created_at) VALUES(?,?,?,?,?) ON CONFLICT(source_id,page) DO NOTHING''', (source_id,page,batch_id,count,db.now())) else: count=existing['input_count'];batch_id=existing['batch_id'] finished=count=source['max_pages'] final_state=('limit_reached' if page>=source['max_pages'] and count>=source['per_page'] else 'completed') if finished else 'ready' with db.connect() as c: c.execute('BEGIN IMMEDIATE') cur=c.execute('SELECT next_page,state FROM v84_api_sources WHERE id=? AND site_id=?',(source_id,site_id)).fetchone() if cur and cur['state']=='cancelled': return {'state':'cancelled','page':page,'count':count,'batch_id':batch_id,'next_page':page} if cur and cur['next_page']==page: c.execute('''UPDATE v84_api_sources SET next_page=?,scanned_pages=scanned_pages+1, staged_count=staged_count+?,state=?,updated_at=?,error='' WHERE id=?''', (page+1,count,final_state,db.now(),source_id)) return {'state':final_state,'page':page,'count':count,'batch_id':batch_id, 'next_page':page+1} def run_scan(site_id,source_id,job_id=None): source=_source(site_id,source_id) if source['state']=='cancelled':return status(site_id,source_id) # Worker retry always resumes from the durable checkpoint. while True: one=scan_page(site_id,source_id) if one['state'] in ('completed','limit_reached'):break if one['state']=='cancelled':break return status(site_id,source_id) def submit_scan(site_id,source_id,actor,key=None): source=_source(site_id,source_id) if source['state']=='cancelled':raise HTTPException(409,'数据源已取消') if key is not None and not (8<=len(key)<=128 and all(c.isalnum() or c in '_-:.' for c in key)): raise ValueError('幂等键不合法') payload=json.dumps({'source_id':source_id},sort_keys=True,separators=(',',':')) with db.connect() as c: c.execute('BEGIN IMMEDIATE') if key: prior=c.execute('SELECT id,state,kind,payload_json FROM build_jobs WHERE site_id=? AND idempotency_key=?',(site_id,key)).fetchone() if prior: if prior['kind']!='api_scan' or prior['payload_json']!=payload:raise ValueError('幂等键已关联其他任务') return {'id':prior['id'],'state':prior['state'],'existing':True} prior=c.execute('''SELECT id,state FROM build_jobs WHERE site_id=? AND kind='api_scan' AND state IN ('queued','running','retrying') AND payload_json=? LIMIT 1''',(site_id,payload)).fetchone() if prior:return {'id':prior['id'],'state':prior['state'],'existing':True} if source['state'] in ('completed','limit_reached'):return {'state':source['state'],'existing':True,'source_id':source_id} if c.execute("SELECT COUNT(*) FROM build_jobs WHERE state IN ('queued','running','retrying')").fetchone()[0]>=jobs._limit('SITEFORGE_MAX_PENDING_JOBS',100): raise jobs.JobCapacityError('全局任务容量已满') if c.execute("SELECT COUNT(*) FROM build_jobs WHERE site_id=? AND state IN ('queued','running','retrying')",(site_id,)).fetchone()[0]>=jobs._limit('SITEFORGE_MAX_PENDING_JOBS_PER_SITE',5): raise jobs.JobCapacityError('当前网站任务容量已满') jid=uuid.uuid4().hex[:16] c.execute('''INSERT INTO build_jobs(id,site_id,kind,state,created_by,created_at, payload_json,idempotency_key,max_attempts,timeout_seconds,available_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)''', (jid,site_id,'api_scan','queued',actor,db.now(),payload,key,3,900,db.now())) from . import redis_signal redis_signal.notify() return {'id':jid,'state':'queued','existing':False} @router.post('/{site_id}/xml/staging') async def api_xml(request:Request,site_id:str,file:UploadFile=File(...),record_tag:str=Form('item'), content_type:str=Form('product'),mapping_json:str=Form('{}')): enterprise.require(request,site_id,'editor') blob=await file.read(MAX_XML_BYTES+1) try: mapping=json.loads(mapping_json) rows=parse_xml_records(blob,record_tag) result=v8_workflows.stage(site_id,rows,mapping or None,content_type,'xml:'+str(file.filename or 'upload.xml')[:100]) return {**result,'columns':list(rows[0]),'record_tag':record_tag} except (ValueError,TypeError,json.JSONDecodeError) as exc:raise HTTPException(400,str(exc)) @router.post('/{site_id}/api-sources') def api_create(request:Request,site_id:str,payload:dict=Body(...)): enterprise.require(request,site_id,'editor') try:return create_api_source(site_id,payload) except ValueError as exc:raise HTTPException(400,str(exc)) @router.get('/{site_id}/api-sources') def api_list(request:Request,site_id:str): enterprise.require(request,site_id,'viewer') v8_workflows._require_site(site_id) return {'items':db.rows('SELECT id,url,state,next_page,staged_count FROM v84_api_sources WHERE site_id=? ORDER BY created_at DESC LIMIT 100',(site_id,))} @router.get('/{site_id}/api-sources/{source_id}') def api_status(request:Request,site_id:str,source_id:str): enterprise.require(request,site_id,'viewer') return status(site_id,source_id) @router.post('/{site_id}/api-sources/{source_id}/scan',status_code=202) def api_scan(request:Request,site_id:str,source_id:str,payload:dict=Body(default_factory=dict)): enterprise.require(request,site_id,'editor') try:return submit_scan(site_id,source_id,request.state.actor,payload.get('idempotency_key')) except (ValueError,jobs.JobCapacityError) as exc:raise HTTPException(409,str(exc)) @router.post('/{site_id}/api-sources/{source_id}/cancel') def api_cancel(request:Request,site_id:str,source_id:str): enterprise.require(request,site_id,'editor') s=_source(site_id,source_id) if s['state']!='completed':db.execute("UPDATE v84_api_sources SET state='cancelled',updated_at=? WHERE id=?",(db.now(),source_id)) for row in db.rows("SELECT id FROM build_jobs WHERE site_id=? AND kind='api_scan' AND state IN ('queued','running','retrying')",(site_id,)): job=db.one('SELECT payload_json FROM build_jobs WHERE id=?',(row['id'],)) if job and json.loads(job['payload_json']).get('source_id')==source_id:jobs.cancel(site_id,row['id']) return status(site_id,source_id)