352 lines
19 KiB
Python
352 lines
19 KiB
Python
|
"""V8.4 bounded XML and public JSON API connectors.
|
||
|
|
|
||
|
|
All resulting content enters the existing staged, auditable V8 import batches.
|
||
|
|
Public API paging runs in a subprocess under the common Worker lease/timeout.
|
||
|
|
Authenticated API sources and database connectors are intentionally not enabled.
|
||
|
|
"""
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import json
|
||
|
|
import ipaddress
|
||
|
|
import re
|
||
|
|
import uuid
|
||
|
|
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||
|
|
from fastapi import APIRouter, Body, File, Form, HTTPException, Request, UploadFile
|
||
|
|
from defusedxml import ElementTree as XML
|
||
|
|
from defusedxml.common import DefusedXmlException
|
||
|
|
from . import db, enterprise, jobs, safe_http, v8_workflows, secret_refs
|
||
|
|
|
||
|
|
router = APIRouter(prefix='/api/connectors/sites',tags=['V8.4 data connectors'])
|
||
|
|
MAX_XML_BYTES = 8_000_000
|
||
|
|
MAX_XML_NODES = 20_000
|
||
|
|
MAX_XML_DEPTH = 9
|
||
|
|
MAX_VALUE_LEN = 20_000
|
||
|
|
MAX_RECORDS = 1000
|
||
|
|
MAX_JSON_BYTES = 2_000_000
|
||
|
|
TAG = re.compile(r'^[A-Za-z_][\w.-]{0,70}$')
|
||
|
|
JSON_PATH = re.compile(r'^[A-Za-z_][\w-]{0,60}(?:\.[A-Za-z_][\w-]{0,60}){0,4}$')
|
||
|
|
SENSITIVE_KEYS = {'key','api_key','apikey','token','access_token','secret','password','authorization','auth','signature','client_secret'}
|
||
|
|
|
||
|
|
|
||
|
|
def setup():
|
||
|
|
with db.connect() as c:
|
||
|
|
c.executescript('''
|
||
|
|
CREATE TABLE IF NOT EXISTS v84_api_sources (
|
||
|
|
id TEXT PRIMARY KEY, site_id TEXT NOT NULL REFERENCES sites(id) ON DELETE CASCADE,
|
||
|
|
url TEXT NOT NULL, items_path TEXT NOT NULL, page_param TEXT NOT NULL,
|
||
|
|
per_page_param TEXT NOT NULL, per_page INTEGER NOT NULL,
|
||
|
|
max_pages INTEGER NOT NULL, next_page INTEGER NOT NULL DEFAULT 1,
|
||
|
|
scanned_pages INTEGER NOT NULL DEFAULT 0, staged_count INTEGER NOT NULL DEFAULT 0,
|
||
|
|
kind TEXT NOT NULL, mapping_json TEXT NOT NULL, state TEXT NOT NULL DEFAULT 'ready',
|
||
|
|
created_at TEXT NOT NULL, updated_at TEXT NOT NULL, error TEXT NOT NULL DEFAULT '');
|
||
|
|
CREATE INDEX IF NOT EXISTS idx_v84_sources_site ON v84_api_sources(site_id,created_at);
|
||
|
|
CREATE TABLE IF NOT EXISTS v84_api_pages (
|
||
|
|
source_id TEXT NOT NULL REFERENCES v84_api_sources(id) ON DELETE CASCADE,
|
||
|
|
page INTEGER NOT NULL, batch_id TEXT REFERENCES v8_import_batches(id),
|
||
|
|
input_count INTEGER NOT NULL, created_at TEXT NOT NULL,
|
||
|
|
PRIMARY KEY(source_id,page));
|
||
|
|
''')
|
||
|
|
# Preserve V8.4 sources when upgrading an existing database.
|
||
|
|
existing={r['name'] for r in c.execute('PRAGMA table_info(v84_api_sources)')}
|
||
|
|
for col,ddl in {
|
||
|
|
'auth_type':"TEXT NOT NULL DEFAULT 'none'",
|
||
|
|
'auth_header':"TEXT NOT NULL DEFAULT ''",
|
||
|
|
'secret_ref':"TEXT NOT NULL DEFAULT ''",
|
||
|
|
}.items():
|
||
|
|
if col not in existing:c.execute(f'ALTER TABLE v84_api_sources ADD COLUMN {col} {ddl}')
|
||
|
|
|
||
|
|
|
||
|
|
def _local_name(tag):
|
||
|
|
return tag.rsplit('}',1)[-1] if isinstance(tag,str) else ''
|
||
|
|
|
||
|
|
|
||
|
|
def parse_xml_records(data:bytes, record_tag:str):
|
||
|
|
"""Parse a bounded, non-DTD, flat/two-level XML dataset without network access."""
|
||
|
|
if not isinstance(data,bytes) or not 0 < len(data) <= MAX_XML_BYTES:
|
||
|
|
raise ValueError('XML 文件大小必须在 1 字节到 8 MB 之间')
|
||
|
|
if not isinstance(record_tag,str) or not TAG.fullmatch(record_tag):
|
||
|
|
raise ValueError('记录节点名称不合法')
|
||
|
|
# defusedxml is the security gate against DTD entity expansion and XXE.
|
||
|
|
try:root=XML.fromstring(data,forbid_dtd=True)
|
||
|
|
except (DefusedXmlException, XML.ParseError) as exc:
|
||
|
|
raise ValueError('XML 包含不允许的实体或格式错误') from exc
|
||
|
|
stack=[(root,1)];nodes=0;records=[]
|
||
|
|
while stack:
|
||
|
|
elem,depth=stack.pop();nodes+=1
|
||
|
|
if nodes>MAX_XML_NODES or depth>MAX_XML_DEPTH:
|
||
|
|
raise ValueError('XML 节点数或嵌套层级超限')
|
||
|
|
if _local_name(elem.tag)==record_tag:
|
||
|
|
records.append(elem)
|
||
|
|
if len(records)>MAX_RECORDS:raise ValueError('XML 每次最多导入1000条记录')
|
||
|
|
for child in reversed(list(elem)):stack.append((child,depth+1))
|
||
|
|
if not records:raise ValueError('未找到指定的 XML 记录节点')
|
||
|
|
rows=[]
|
||
|
|
for record in records:
|
||
|
|
item={}
|
||
|
|
for child in record:
|
||
|
|
field=_local_name(child.tag)
|
||
|
|
if not TAG.fullmatch(field):raise ValueError('XML 字段名不合法')
|
||
|
|
if len(child):
|
||
|
|
for sub in child:
|
||
|
|
if len(sub):raise ValueError('仅支持扁平或两层结构的记录')
|
||
|
|
key=field+'.'+_local_name(sub.tag)
|
||
|
|
if key in item:raise ValueError('XML 记录存在重复字段：'+key)
|
||
|
|
item[key]=(sub.text or '').strip()[:MAX_VALUE_LEN]
|
||
|
|
else:
|
||
|
|
if field in item:raise ValueError('XML 记录存在重复字段：'+field)
|
||
|
|
item[field]=(child.text or '').strip()[:MAX_VALUE_LEN]
|
||
|
|
if item:rows.append(item)
|
||
|
|
if not rows:raise ValueError('XML 记录没有可导入字段')
|
||
|
|
if len(json.dumps(rows,ensure_ascii=False).encode('utf-8'))>v8_workflows.MAX_UPLOAD:
|
||
|
|
raise ValueError('XML 解码后数据超过导入限制')
|
||
|
|
return rows
|
||
|
|
|
||
|
|
|
||
|
|
def _validate_url(url):
|
||
|
|
if not isinstance(url,str) or len(url)>1600:raise ValueError('API 地址不合法')
|
||
|
|
p=urlsplit(url)
|
||
|
|
if p.scheme!='https' or not p.hostname or p.username or p.password or p.fragment:
|
||
|
|
raise ValueError('API 连接器只接受公开 HTTPS 地址，不能包含账号、口令或片段')
|
||
|
|
try:port=p.port
|
||
|
|
except ValueError as exc:raise ValueError('API 端口不合法') from exc
|
||
|
|
if port not in (None,443):raise ValueError('仅支持默认 HTTPS 端口')
|
||
|
|
try:
|
||
|
|
ip=ipaddress.ip_address(p.hostname)
|
||
|
|
if not ip.is_global:raise ValueError('禁止使用非公网 IP API 地址')
|
||
|
|
except ValueError as exc:
|
||
|
|
if str(exc)=='禁止使用非公网 IP API 地址':raise
|
||
|
|
if any(k.casefold() in SENSITIVE_KEYS or 'token' in k.casefold() or 'secret' in k.casefold()
|
||
|
|
for k,_ in parse_qsl(p.query,keep_blank_values=True)):
|
||
|
|
raise ValueError('请勿在 API URL 中传入凭据或令牌')
|
||
|
|
if len(parse_qsl(p.query,keep_blank_values=True))>30:raise ValueError('URL 查询参数过多')
|
||
|
|
# DNS->socket pinning and non-public IP protection are enforced on fetch.
|
||
|
|
return urlunsplit(p)
|
||
|
|
|
||
|
|
|
||
|
|
def create_api_source(site_id,payload):
|
||
|
|
v8_workflows._require_site(site_id)
|
||
|
|
if not isinstance(payload,dict):raise ValueError('API 参数必须是对象')
|
||
|
|
url=_validate_url(payload.get('url'))
|
||
|
|
kind=str(payload.get('content_type','product'))
|
||
|
|
if kind not in ('product','vehicle','page','post','brand','case','service','other'):
|
||
|
|
raise ValueError('内容类型不受支持')
|
||
|
|
items_path=str(payload.get('items_path','items'))
|
||
|
|
if not JSON_PATH.fullmatch(items_path):raise ValueError('JSON 路径不合法')
|
||
|
|
page_param=str(payload.get('page_param','page'))
|
||
|
|
per_page_param=str(payload.get('per_page_param','per_page'))
|
||
|
|
if not TAG.fullmatch(page_param) or not TAG.fullmatch(per_page_param) or page_param==per_page_param:
|
||
|
|
raise ValueError('分页参数不合法')
|
||
|
|
if any(k in (page_param,per_page_param) for k,_ in parse_qsl(urlsplit(url).query)):
|
||
|
|
raise ValueError('源 URL 不应携带分页参数')
|
||
|
|
try:page_size=int(payload.get('page_size',50));max_pages=int(payload.get('max_pages',20))
|
||
|
|
except (TypeError,ValueError):raise ValueError('分页数量不合法')
|
||
|
|
if not 1<=page_size<=100 or not 1<=max_pages<=50 or page_size*max_pages>5000:
|
||
|
|
raise ValueError('上限为每页100条、最多50页、合计5000条')
|
||
|
|
mapping=payload.get('mapping') or {}
|
||
|
|
if not isinstance(mapping,dict) or len(mapping)>100 or any(not isinstance(k,str) or not isinstance(v,str) or len(v)>100 for k,v in mapping.items()):
|
||
|
|
raise ValueError('字段映射规则不合法')
|
||
|
|
auth_type=str(payload.get('auth_type','none'))
|
||
|
|
if auth_type not in ('none','bearer','api_key','custom'):
|
||
|
|
raise ValueError('不支持的 API 认证方式')
|
||
|
|
secret_ref='';auth_header=''
|
||
|
|
if auth_type!='none':
|
||
|
|
secret_ref=secret_refs.validate_reference(payload.get('secret_ref'))
|
||
|
|
auth_header='Authorization' if auth_type=='bearer' else 'X-API-Key'
|
||
|
|
if auth_type=='custom':
|
||
|
|
auth_header=str(payload.get('auth_header',''))
|
||
|
|
if not re.fullmatch(r'X-[A-Za-z0-9-]{2,64}',auth_header,re.I):
|
||
|
|
raise ValueError('自定义凭据请求头必须以 X- 开头')
|
||
|
|
elif payload.get('secret_ref'):
|
||
|
|
raise ValueError('公开 API 不应保存密钥引用')
|
||
|
|
source_id=uuid.uuid4().hex
|
||
|
|
db.insert('''INSERT INTO v84_api_sources(id,site_id,url,items_path,page_param,per_page_param,
|
||
|
|
per_page,max_pages,kind,mapping_json,auth_type,auth_header,secret_ref,created_at,updated_at)
|
||
|
|
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''',
|
||
|
|
(source_id,site_id,url,items_path,page_param,per_page_param,page_size,max_pages,kind,
|
||
|
|
json.dumps(mapping,ensure_ascii=False),auth_type,auth_header,secret_ref,db.now(),db.now()))
|
||
|
|
return status(site_id,source_id)
|
||
|
|
|
||
|
|
|
||
|
|
def _source(site_id,source_id):
|
||
|
|
r=db.one('SELECT * FROM v84_api_sources WHERE id=? AND site_id=?',(source_id,site_id))
|
||
|
|
if not r:raise HTTPException(404,'API 数据源不存在')
|
||
|
|
return r
|
||
|
|
|
||
|
|
|
||
|
|
def status(site_id,source_id):
|
||
|
|
s=_source(site_id,source_id)
|
||
|
|
# Secret references are never returned by management APIs.
|
||
|
|
return {k:s[k] for k in ('id','site_id','url','items_path','state','next_page',
|
||
|
|
'scanned_pages','staged_count','max_pages','per_page','kind','error','auth_type','auth_header')} | {
|
||
|
|
'batches':db.rows('SELECT page,batch_id,input_count FROM v84_api_pages WHERE source_id=? ORDER BY page',(source_id,))}
|
||
|
|
|
||
|
|
|
||
|
|
def _data_items(content,items_path):
|
||
|
|
try:result=json.loads(content.decode('utf-8-sig'))
|
||
|
|
except (ValueError,UnicodeError) as exc:raise ValueError('API 响应不是有效 UTF-8 JSON') from exc
|
||
|
|
if isinstance(result,list):items=result
|
||
|
|
else:
|
||
|
|
for part in items_path.split('.'):
|
||
|
|
if not isinstance(result,dict) or part not in result:raise ValueError('API JSON 路径不存在：'+items_path)
|
||
|
|
result=result[part]
|
||
|
|
items=result
|
||
|
|
if not isinstance(items,list) or len(items)>100 or any(not isinstance(x,dict) for x in items):
|
||
|
|
raise ValueError('API 当前页必须包含最多100条对象记录')
|
||
|
|
if any(len(row)>150 for row in items):raise ValueError('API 单条字段数量超过限制')
|
||
|
|
return items
|
||
|
|
|
||
|
|
|
||
|
|
def page_url(source,page):
|
||
|
|
p=urlsplit(source['url']);params=parse_qsl(p.query,keep_blank_values=True)
|
||
|
|
params.extend(((source['page_param'],str(page)),(source['per_page_param'],str(source['per_page']))))
|
||
|
|
return urlunsplit((p.scheme,p.netloc,p.path,urlencode(params),''))
|
||
|
|
|
||
|
|
|
||
|
|
def scan_page(site_id,source_id):
|
||
|
|
source=_source(site_id,source_id)
|
||
|
|
if source['state'] in ('cancelled','completed','limit_reached'):return {'state':source['state'],'next_page':source['next_page']}
|
||
|
|
if source['next_page']>source['max_pages']:
|
||
|
|
db.execute("UPDATE v84_api_sources SET state='completed',updated_at=? WHERE id=?",(db.now(),source_id))
|
||
|
|
return {'state':'completed','next_page':source['next_page']}
|
||
|
|
page=int(source['next_page'])
|
||
|
|
# A process crash after stage but before the checkpoint must not stage twice.
|
||
|
|
existing=db.one('SELECT * FROM v84_api_pages WHERE source_id=? AND page=?',(source_id,page))
|
||
|
|
origin=f'api:{source_id}:page:{page}'
|
||
|
|
if not existing:
|
||
|
|
pending=db.one('SELECT id,input_count FROM v8_import_batches WHERE site_id=? AND origin=?',(site_id,origin))
|
||
|
|
if pending:
|
||
|
|
batch_id=pending['id'];count=pending['input_count']
|
||
|
|
else:
|
||
|
|
extra={}
|
||
|
|
if source.get('auth_type','none')!='none':
|
||
|
|
token=secret_refs.resolve(source['secret_ref'])
|
||
|
|
extra[source['auth_header']]=('Bearer '+token if source['auth_type']=='bearer' else token)
|
||
|
|
opts={'headers':{'Accept':'application/json'},'redirects':0,'timeout':20,'max_bytes':MAX_JSON_BYTES}
|
||
|
|
if extra:opts['credential_headers']=extra
|
||
|
|
resp=safe_http.fetch(page_url(source,page),**opts)
|
||
|
|
resp.raise_for_status()
|
||
|
|
mime=resp.headers.get('content-type','').split(';')[0].strip().lower()
|
||
|
|
if mime!='application/json' and not mime.endswith('+json'):
|
||
|
|
raise ValueError('API 响应 Content-Type 必须为 JSON')
|
||
|
|
rows=_data_items(resp.content,source['items_path'])
|
||
|
|
count=len(rows);batch_id=None
|
||
|
|
if rows:
|
||
|
|
mapping=json.loads(source['mapping_json'] or '{}') or None
|
||
|
|
keys=[f'{source_id}:{page}:{i}:{str(r.get("id",i))[:60]}' for i,r in enumerate(rows)]
|
||
|
|
staged=v8_workflows.stage(site_id,rows,mapping,source['kind'],origin,keys)
|
||
|
|
batch_id=staged['batch_id']
|
||
|
|
with db.connect() as c:
|
||
|
|
c.execute('BEGIN IMMEDIATE')
|
||
|
|
# The exclusive one-page checkpoint is idempotent across retries.
|
||
|
|
c.execute('''INSERT INTO v84_api_pages(source_id,page,batch_id,input_count,created_at)
|
||
|
|
VALUES(?,?,?,?,?) ON CONFLICT(source_id,page) DO NOTHING''',
|
||
|
|
(source_id,page,batch_id,count,db.now()))
|
||
|
|
else:
|
||
|
|
count=existing['input_count'];batch_id=existing['batch_id']
|
||
|
|
finished=count<source['per_page'] or page>=source['max_pages']
|
||
|
|
final_state=('limit_reached' if page>=source['max_pages'] and count>=source['per_page'] else 'completed') if finished else 'ready'
|
||
|
|
with db.connect() as c:
|
||
|
|
c.execute('BEGIN IMMEDIATE')
|
||
|
|
cur=c.execute('SELECT next_page,state FROM v84_api_sources WHERE id=? AND site_id=?',(source_id,site_id)).fetchone()
|
||
|
|
if cur and cur['state']=='cancelled':
|
||
|
|
return {'state':'cancelled','page':page,'count':count,'batch_id':batch_id,'next_page':page}
|
||
|
|
if cur and cur['next_page']==page:
|
||
|
|
c.execute('''UPDATE v84_api_sources SET next_page=?,scanned_pages=scanned_pages+1,
|
||
|
|
staged_count=staged_count+?,state=?,updated_at=?,error='' WHERE id=?''',
|
||
|
|
(page+1,count,final_state,db.now(),source_id))
|
||
|
|
return {'state':final_state,'page':page,'count':count,'batch_id':batch_id,
|
||
|
|
'next_page':page+1}
|
||
|
|
|
||
|
|
|
||
|
|
def run_scan(site_id,source_id,job_id=None):
|
||
|
|
source=_source(site_id,source_id)
|
||
|
|
if source['state']=='cancelled':return status(site_id,source_id)
|
||
|
|
# Worker retry always resumes from the durable checkpoint.
|
||
|
|
while True:
|
||
|
|
one=scan_page(site_id,source_id)
|
||
|
|
if one['state'] in ('completed','limit_reached'):break
|
||
|
|
if one['state']=='cancelled':break
|
||
|
|
return status(site_id,source_id)
|
||
|
|
|
||
|
|
|
||
|
|
def submit_scan(site_id,source_id,actor,key=None):
|
||
|
|
source=_source(site_id,source_id)
|
||
|
|
if source['state']=='cancelled':raise HTTPException(409,'数据源已取消')
|
||
|
|
if key is not None and not (8<=len(key)<=128 and all(c.isalnum() or c in '_-:.' for c in key)):
|
||
|
|
raise ValueError('幂等键不合法')
|
||
|
|
payload=json.dumps({'source_id':source_id},sort_keys=True,separators=(',',':'))
|
||
|
|
with db.connect() as c:
|
||
|
|
c.execute('BEGIN IMMEDIATE')
|
||
|
|
if key:
|
||
|
|
prior=c.execute('SELECT id,state,kind,payload_json FROM build_jobs WHERE site_id=? AND idempotency_key=?',(site_id,key)).fetchone()
|
||
|
|
if prior:
|
||
|
|
if prior['kind']!='api_scan' or prior['payload_json']!=payload:raise ValueError('幂等键已关联其他任务')
|
||
|
|
return {'id':prior['id'],'state':prior['state'],'existing':True}
|
||
|
|
prior=c.execute('''SELECT id,state FROM build_jobs WHERE site_id=? AND kind='api_scan'
|
||
|
|
AND state IN ('queued','running','retrying') AND payload_json=? LIMIT 1''',(site_id,payload)).fetchone()
|
||
|
|
if prior:return {'id':prior['id'],'state':prior['state'],'existing':True}
|
||
|
|
if source['state'] in ('completed','limit_reached'):return {'state':source['state'],'existing':True,'source_id':source_id}
|
||
|
|
if c.execute("SELECT COUNT(*) FROM build_jobs WHERE state IN ('queued','running','retrying')").fetchone()[0]>=jobs._limit('SITEFORGE_MAX_PENDING_JOBS',100):
|
||
|
|
raise jobs.JobCapacityError('全局任务容量已满')
|
||
|
|
if c.execute("SELECT COUNT(*) FROM build_jobs WHERE site_id=? AND state IN ('queued','running','retrying')",(site_id,)).fetchone()[0]>=jobs._limit('SITEFORGE_MAX_PENDING_JOBS_PER_SITE',5):
|
||
|
|
raise jobs.JobCapacityError('当前网站任务容量已满')
|
||
|
|
jid=uuid.uuid4().hex[:16]
|
||
|
|
c.execute('''INSERT INTO build_jobs(id,site_id,kind,state,created_by,created_at,
|
||
|
|
payload_json,idempotency_key,max_attempts,timeout_seconds,available_at)
|
||
|
|
VALUES(?,?,?,?,?,?,?,?,?,?,?)''',
|
||
|
|
(jid,site_id,'api_scan','queued',actor,db.now(),payload,key,3,900,db.now()))
|
||
|
|
from . import redis_signal
|
||
|
|
redis_signal.notify()
|
||
|
|
return {'id':jid,'state':'queued','existing':False}
|
||
|
|
|
||
|
|
|
||
|
|
@router.post('/{site_id}/xml/staging')
|
||
|
|
async def api_xml(request:Request,site_id:str,file:UploadFile=File(...),record_tag:str=Form('item'),
|
||
|
|
content_type:str=Form('product'),mapping_json:str=Form('{}')):
|
||
|
|
enterprise.require(request,site_id,'editor')
|
||
|
|
blob=await file.read(MAX_XML_BYTES+1)
|
||
|
|
try:
|
||
|
|
mapping=json.loads(mapping_json)
|
||
|
|
rows=parse_xml_records(blob,record_tag)
|
||
|
|
result=v8_workflows.stage(site_id,rows,mapping or None,content_type,'xml:'+str(file.filename or 'upload.xml')[:100])
|
||
|
|
return {**result,'columns':list(rows[0]),'record_tag':record_tag}
|
||
|
|
except (ValueError,TypeError,json.JSONDecodeError) as exc:raise HTTPException(400,str(exc))
|
||
|
|
|
||
|
|
|
||
|
|
@router.post('/{site_id}/api-sources')
|
||
|
|
def api_create(request:Request,site_id:str,payload:dict=Body(...)):
|
||
|
|
enterprise.require(request,site_id,'editor')
|
||
|
|
try:return create_api_source(site_id,payload)
|
||
|
|
except ValueError as exc:raise HTTPException(400,str(exc))
|
||
|
|
|
||
|
|
|
||
|
|
@router.get('/{site_id}/api-sources')
|
||
|
|
def api_list(request:Request,site_id:str):
|
||
|
|
enterprise.require(request,site_id,'viewer')
|
||
|
|
v8_workflows._require_site(site_id)
|
||
|
|
return {'items':db.rows('SELECT id,url,state,next_page,staged_count FROM v84_api_sources WHERE site_id=? ORDER BY created_at DESC LIMIT 100',(site_id,))}
|
||
|
|
|
||
|
|
|
||
|
|
@router.get('/{site_id}/api-sources/{source_id}')
|
||
|
|
def api_status(request:Request,site_id:str,source_id:str):
|
||
|
|
enterprise.require(request,site_id,'viewer')
|
||
|
|
return status(site_id,source_id)
|
||
|
|
|
||
|
|
|
||
|
|
@router.post('/{site_id}/api-sources/{source_id}/scan',status_code=202)
|
||
|
|
def api_scan(request:Request,site_id:str,source_id:str,payload:dict=Body(default_factory=dict)):
|
||
|
|
enterprise.require(request,site_id,'editor')
|
||
|
|
try:return submit_scan(site_id,source_id,request.state.actor,payload.get('idempotency_key'))
|
||
|
|
except (ValueError,jobs.JobCapacityError) as exc:raise HTTPException(409,str(exc))
|
||
|
|
|
||
|
|
|
||
|
|
@router.post('/{site_id}/api-sources/{source_id}/cancel')
|
||
|
|
def api_cancel(request:Request,site_id:str,source_id:str):
|
||
|
|
enterprise.require(request,site_id,'editor')
|
||
|
|
s=_source(site_id,source_id)
|
||
|
|
if s['state']!='completed':db.execute("UPDATE v84_api_sources SET state='cancelled',updated_at=? WHERE id=?",(db.now(),source_id))
|
||
|
|
for row in db.rows("SELECT id FROM build_jobs WHERE site_id=? AND kind='api_scan' AND state IN ('queued','running','retrying')",(site_id,)):
|
||
|
|
job=db.one('SELECT payload_json FROM build_jobs WHERE id=?',(row['id'],))
|
||
|
|
if job and json.loads(job['payload_json']).get('source_id')==source_id:jobs.cancel(site_id,row['id'])
|
||
|
|
return status(site_id,source_id)
|