feat(deploy): Docker镜像支持页面抓取(Playwright+Chromium+scripts/)+full_sync可移植
- Dockerfile:Chromium headless 系统依赖 + playwright install chromium + COPY scripts/(调度器 _page_subprocess 的入口,之前镜像里没有) - full_sync_all.sh:DB_URL 外部可覆盖(容器 MySQL)、python 路径可移植 (容器 PATH / 本地 venv 双兼容) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
+29
-1
@@ -9,11 +9,34 @@ FROM python:3.12-slim
|
|||||||
|
|
||||||
# Build-time / runtime deps for asyncmy (compiled C extension) + httpx +
|
# Build-time / runtime deps for asyncmy (compiled C extension) + httpx +
|
||||||
# cryptography.``libmariadb`` is what asyncmy links against on Debian.
|
# cryptography.``libmariadb`` is what asyncmy links against on Debian.
|
||||||
|
# Playwright 系统依赖:Chromium headless 需要的共享库(libnss3/libatk/libx11
|
||||||
|
# 等,playwright install-deps 的等价精简集,2026-08-21 部署阻塞点#1)。
|
||||||
RUN apt-get update \
|
RUN apt-get update \
|
||||||
&& apt-get install -y --no-install-recommends \
|
&& apt-get install -y --no-install-recommends \
|
||||||
build-essential \
|
build-essential \
|
||||||
libmariadb-dev \
|
libmariadb-dev \
|
||||||
libffi-dev \
|
libffi-dev \
|
||||||
|
# Chromium headless 依赖(playwright 官方 install-deps 清单)
|
||||||
|
libnss3 \
|
||||||
|
libnspr4 \
|
||||||
|
libatk1.0-0 \
|
||||||
|
libatk-bridge2.0-0 \
|
||||||
|
libcups2 \
|
||||||
|
libdrm2 \
|
||||||
|
libxkbcommon0 \
|
||||||
|
libxcomposite1 \
|
||||||
|
libxdamage1 \
|
||||||
|
libxfixes3 \
|
||||||
|
libxrandr2 \
|
||||||
|
libgbm1 \
|
||||||
|
libasound2 \
|
||||||
|
libpango-1.0-0 \
|
||||||
|
libcairo2 \
|
||||||
|
libatspi2.0-0 \
|
||||||
|
libx11-6 \
|
||||||
|
libxcb1 \
|
||||||
|
libxext6 \
|
||||||
|
libexpat1 \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
@@ -24,11 +47,16 @@ WORKDIR /app
|
|||||||
COPY pyproject.toml ./
|
COPY pyproject.toml ./
|
||||||
COPY app/__init__.py app/__init__.py
|
COPY app/__init__.py app/__init__.py
|
||||||
COPY app/ ./app/
|
COPY app/ ./app/
|
||||||
|
# scripts/ — 调度器的页面抓取入口(scripts/scrape_via_page.py 与
|
||||||
|
# full_sync_all.sh 由 scheduler._page_subprocess 以 cwd=/app 调起,
|
||||||
|
# 2026-08-21 部署阻塞点#2:镜像里必须带上)。
|
||||||
|
COPY scripts/ ./scripts/
|
||||||
|
|
||||||
# Install the package + dev dependencies (the dev extras include aiosqlite +
|
# Install the package + dev dependencies (the dev extras include aiosqlite +
|
||||||
# pytest, which are tiny and convenient for in-container smoke checks). Pin
|
# pytest, which are tiny and convenient for in-container smoke checks). Pin
|
||||||
# the install to the local source so the package import path resolves.
|
# the install to the local source so the package import path resolves.
|
||||||
RUN pip install --no-cache-dir -e ".[dev]"
|
RUN pip install --no-cache-dir -e ".[dev]" \
|
||||||
|
&& playwright install chromium --with-deps
|
||||||
|
|
||||||
# Default command — overridden per-service in docker-compose. The API service
|
# Default command — overridden per-service in docker-compose. The API service
|
||||||
# runs uvicorn; the scheduler service runs the CLI.
|
# runs uvicorn; the scheduler service runs the CLI.
|
||||||
|
|||||||
@@ -1,8 +1,14 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
# FULL page-based re-sync: ALL channels, ALL entities — ZERO API calls, ZERO 风控
|
# FULL page-based re-sync: ALL channels, ALL entities — ZERO API calls, ZERO 风控
|
||||||
cd "$(dirname "$0")/.."
|
cd "$(dirname "$0")/.."
|
||||||
export DB_URL="sqlite+aiosqlite:///./smoke.db"
|
# DB_URL:外部已设则沿用(容器里由 docker-compose 注入 MySQL);本地默认 SQLite
|
||||||
PY=".venv/Scripts/python.exe"
|
export DB_URL="${DB_URL:-sqlite+aiosqlite:///./smoke.db}"
|
||||||
|
# 可移植 python:容器里 python 在 PATH;本地 Windows 用 venv 的解释器(2026-08-21 部署阻塞点#3)
|
||||||
|
if [ -f ".venv/Scripts/python.exe" ]; then
|
||||||
|
PY=".venv/Scripts/python.exe"
|
||||||
|
else
|
||||||
|
PY="python"
|
||||||
|
fi
|
||||||
|
|
||||||
echo "=============================================="
|
echo "=============================================="
|
||||||
echo "$(date) === FULL PAGE-BASED RE-SYNC (NO API) ==="
|
echo "$(date) === FULL PAGE-BASED RE-SYNC (NO API) ==="
|
||||||
|
|||||||
Reference in New Issue
Block a user