fix(security): 路径穿越防护+导出行数上限+并发同步锁(3 critical/important)

- media.py: GET /api/media/{hash} 现强制 file 必须位于 Settings.media_dir
  沙箱内(resolve + relative_to),杜绝 local_path 越权读取任意文件。
- export.py: /api/export/chat-html 加 EXPORT_MAX_ROWS=5000 硬上限(.limit()),
  超出时仅渲染前 5000 条并插入 .export-warning 横幅,防止 OOM。
- scheduler.py: _safe_run 增加 SyncRun advisory lock(60min TTL),发现同
  entity 存在 status='running' 且在 TTL 内的记录即返回 skipped SyncResult;
  同时为本轮 stake 一行 wrapping SyncRun,让并发 CLI/scheduler/手动触发
  能相互避让。崩溃场景由 TTL 自动恢复。
- tests: 新增 path-traversal 拒绝/export 截断警告/advisory lock skip+TTL
  恢复 5 个测试;test_api_media 的 fixture 改为 MEDIA_DIR=tmp_path 以适配
  新沙箱语义。

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-12 10:37:13 +08:00
parent 461e9ff081
commit caef32891e
6 changed files with 388 additions and 6 deletions
+84
View File
@@ -50,6 +50,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_asyn
from app.config import get_settings
from app.db import models # noqa: F401 (register tables on Base.metadata)
from app.db.base import Base
from app.db.models import SyncRun
from app.session.yunke_client import YunkeClient
from app.storage.object_store import ObjectStore
from app.sync.orchestrator import (
@@ -83,6 +84,14 @@ __all__ = ["SyncScheduler", "ENTITY_FUNCS"]
# request, so 200 conversations × N pages each can still take a while).
DEFAULT_MAX_CONVERSATIONS_PER_RUN = 200
# Advisory-lock TTL: a ``running`` SyncRun row older than this is considered
# stale (the process probably crashed mid-run) and a new invocation may
# proceed despite it. 60 min is generous — any entity fits inside — but
# short enough that a crashed run auto-recovers within an hour. Tuned to
# protect against concurrent CLI + scheduler invocations hammering 云控
# with the same scope twice.
ADVISORY_LOCK_TTL_MINUTES = 60
# ---------------------------------------------------------------------------
# Conversation discovery (drives the incremental message sync)
@@ -561,6 +570,15 @@ class SyncScheduler:
This is the failure-policy wrapper that protects the scheduler loop:
a raised exception becomes a logged error and an empty result list,
and the scheduler keeps its next appointment.
Concurrency: an advisory lock refuses to run if another ``_safe_run``
for the same entity is still active. The lock is implemented by
staking a ``running`` SyncRun row keyed on the scheduler entity name
(e.g. ``"all"`` / ``"messages"`` / ``"friends"``) at the start and
finalizing it on completion. A concurrent invocation sees the row and
skips. The 60-min TTL (:data:`ADVISORY_LOCK_TTL_MINUTES`) auto-recovers
from a crashed process: a stale ``running`` row eventually falls
outside the window and a new run may proceed.
"""
started_at = dt.datetime.now(dt.UTC)
client: YunkeClient | None = None
@@ -568,8 +586,74 @@ class SyncScheduler:
try:
client = await _build_client_from_env()
async with self._session_maker() as session:
# --- Advisory lock check ------------------------------
# ``SyncRun.started_at`` is a non-tz-aware DateTime; reading
# a row back yields a naive datetime, so the cutoff is built
# naive-UTC to keep the comparison type-consistent.
cutoff = (
dt.datetime.now(dt.UTC).replace(tzinfo=None)
- dt.timedelta(minutes=ADVISORY_LOCK_TTL_MINUTES)
)
existing = (
await session.execute(
sa.select(SyncRun)
.where(SyncRun.entity_type == entity)
.where(SyncRun.status == "running")
.where(SyncRun.started_at > cutoff)
.limit(1)
)
).scalar_one_or_none()
if existing is not None:
logger.warning(
"sync %s: skipped — another run is active"
" (started %s)",
entity,
existing.started_at.isoformat(),
)
skipped = SyncResult(
entity=entity,
status="skipped",
fetched=0,
upserted=0,
error=f"another sync is running for {entity}",
)
if log:
_log_run(entity, [skipped], started_at=started_at)
return [skipped]
# --- Stake our own 'running' row ----------------------
# A wrapping SyncRun at the scheduler-entity level (distinct
# from any sub-entity rows the orchestrator creates internally).
# On a caught exception the session rolls back and this row
# disappears with everything else — no phantom to recover
# from. On the success path the row is finalized + committed
# below. If the process hard-crashes between flush and commit,
# the row stays 'running' and blocks retries for up to TTL.
run = SyncRun(
entity_type=entity,
status="running",
started_at=dt.datetime.now(dt.UTC),
)
session.add(run)
await session.flush()
runner = ENTITY_FUNCS[entity]
results = await runner(session, client, ObjectStore())
# --- Finalize the wrapping SyncRun --------------------
run.status = (
"success"
if all(r.status == "success" for r in results)
else "failed"
)
run.records_fetched = sum(r.fetched for r in results)
run.records_upserted = sum(r.upserted for r in results)
run.finished_at = dt.datetime.now(dt.UTC)
if run.status == "failed":
joined = "; ".join(
filter(None, (r.error for r in results))
)
run.error = joined or None
await session.commit()
except Exception as exc: # noqa: BLE001 — scheduler MUST NOT crash
logger.exception("sync %s failed", entity)