feat: scheduler + CLI + docker-compose + README
M6 — the scheduling/deployment layer. Makes the system runnable as a
scheduled service + deployable via Docker.
Part A — SyncScheduler (app/sync/scheduler.py):
- APScheduler (AsyncIOScheduler) wraps the orchestrator's sync_* functions.
- Daily full sync at FULL_SYNC_HOUR (default 02:00 local) + incremental
message sync every MESSAGE_INTERVAL_HOURS (default 4).
- ENTITY_FUNCS registry dispatches run_once("<entity>") to the right runner
(departments/accounts/friends/groups/voice_calls/phone_calls/sms/messages/
all_wechat/all).
- Conversations for the message run are discovered from the local customer
mirror (no extra upstream call).
- Failure policy: every run is wrapped — a blown entity is logged + swallowed
so the scheduler keeps its next appointment.
- Per-run JSON-line log (level=info/warning/error, totals, failed entities).
Part B — CLI entry point (app/sync/cli.py):
- `python -m app.sync.cli sync [--entity <name>] [--quiet]` — one-shot sync.
- `python -m app.sync.cli scheduler [--full-sync-hour H] [--message-interval-hours N]`
— long-running scheduler (blocks until SIGINT/SIGTERM).
- Uses _push_settings_into_environ + build_client_from_env (session-cookie
mode preferred; pcLogin deferred).
- Base.metadata.create_all on every command (fresh-checkout friendly).
- Exit codes: 0 success, 1 generic failure, 2 bad CLI / auth misconfig.
Part C — Docker stack:
- yunke-server/Dockerfile: python:3.12-slim, installs deps + app, single
image serves both API (uvicorn) + scheduler (CLI).
- yunkes/Dockerfile: multi-stage node:20 build → nginx serves dist/ +
reverse-proxies /api → backend:8000.
- yunkes/nginx.conf: /api proxy with proxy_buffering off (streaming media),
SPA fallback, asset caching.
- docker-compose.yml: mysql (8.0, healthcheck, volume) + redis (7-alpine) +
backend + scheduler + frontend (port 5173 → 80). Env-driven creds + cadence.
Part D — README.md at repo root: dev mode + docker mode quick-start, config
table, architecture overview, current limitations (pcLogin deferred, WA voice/
media deferred, conservative pacing, conversation discovery is local-only).
Verification:
- pytest: 323 passed (was 294; +29 new for scheduler + CLI).
- npm run build: clean.
- python -m app.sync.cli --help: prints usage.
- docker compose config: VALID.
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -0,0 +1,163 @@
|
||||
# docker-compose for the yunke data mirror (M6, Part C).
|
||||
#
|
||||
# Five services:
|
||||
# mysql — MySQL 8 data store (volume-backed).
|
||||
# redis — Redis 7 (future cache/queue use; not yet wired into the app).
|
||||
# backend — FastAPI viewing API (uvicorn) talking to mysql + redis.
|
||||
# scheduler — APScheduler loop (python -m app.sync.cli scheduler) that
|
||||
# periodically scrapes yunke CRM into mysql.
|
||||
# frontend — yunkes SPA built with Vite, served by nginx, reverse-proxying
|
||||
# /api to the backend service.
|
||||
#
|
||||
# Auth: the scheduler + backend read yunke creds from .env (the host's .env
|
||||
# is passed through as-is). Session-cookie mode (YK_USERID / YK_COMPANY /
|
||||
# YK_COOKIE) is preferred; pcLogin mode is wired but deferred (see README).
|
||||
#
|
||||
# Usage:
|
||||
# docker-compose --env-file .env up # build + start everything
|
||||
# docker-compose up backend # just backend (+ deps)
|
||||
# docker-compose logs -f scheduler # tail scheduler JSON logs
|
||||
#
|
||||
# The frontend's container exposes port 80; mapped to 5173 on the host so
|
||||
# the same URL (http://localhost:5173) works in dev and prod.
|
||||
|
||||
services:
|
||||
# ----------------------------------------------------------------------- #
|
||||
# Data plane
|
||||
# ----------------------------------------------------------------------- #
|
||||
mysql:
|
||||
image: mysql:8.0
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
MYSQL_DATABASE: ${MYSQL_DATABASE:-yunke}
|
||||
MYSQL_USER: ${MYSQL_USER:-yunke}
|
||||
MYSQL_PASSWORD: ${MYSQL_PASSWORD:-yunke}
|
||||
MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-root}
|
||||
# Default MySQL settings are fine for dev. The container ships its own
|
||||
# healthcheck (mysqladmin ping); we use it so dependents wait until the
|
||||
# server is actually accepting connections, not just listening on the port.
|
||||
healthcheck:
|
||||
test: ["CMD", "mysqladmin", "ping", "-h", "localhost"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
volumes:
|
||||
- mysql_data:/var/lib/mysql
|
||||
ports:
|
||||
# Expose for host-side inspection (mysql CLI / DBeaver); omit if you
|
||||
# don't need cross-container access from the host.
|
||||
- "3306:3306"
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
|
||||
# ----------------------------------------------------------------------- #
|
||||
# Application plane — same image, two entrypoints
|
||||
# ----------------------------------------------------------------------- #
|
||||
backend:
|
||||
build:
|
||||
context: ./yunke-server
|
||||
dockerfile: Dockerfile
|
||||
restart: unless-stopped
|
||||
# uvicorn entrypoint (matches the Dockerfile's default CMD, restated so
|
||||
# the role of this service is self-documenting at a glance).
|
||||
command:
|
||||
- uvicorn
|
||||
- app.api.main:app
|
||||
- --host
|
||||
- 0.0.0.0
|
||||
- --port
|
||||
- "8000"
|
||||
depends_on:
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
# DB_URL points at the mysql *service* (not localhost) — the container
|
||||
# resolves it via docker-compose DNS. asyncmy is the async driver the
|
||||
# app expects; the creds match the mysql service block above.
|
||||
DB_URL: mysql+asyncmy://${MYSQL_USER:-yunke}:${MYSQL_PASSWORD:-yunke}@mysql:3306/${MYSQL_DATABASE:-yunke}
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
MEDIA_DIR: /data/media
|
||||
# Pass through yunke auth + viewing-API token verbatim from .env so the
|
||||
# same .env works in dev and docker.
|
||||
YUNKE_BASE_URL: ${YUNKE_BASE_URL:-https://crm2.yunkecn.com}
|
||||
YUNKE_USERNAME: ${YUNKE_USERNAME:-}
|
||||
YUNKE_PASSWORD: ${YUNKE_PASSWORD:-}
|
||||
YK_USERID: ${YK_USERID:-}
|
||||
YK_COMPANY: ${YK_COMPANY:-}
|
||||
YK_COOKIE: ${YK_COOKIE:-}
|
||||
API_TOKEN: ${API_TOKEN:-}
|
||||
ENCRYPTION_KEY: ${ENCRYPTION_KEY:-}
|
||||
volumes:
|
||||
# Persist downloaded media blobs across container restarts.
|
||||
- media_data:/data/media
|
||||
ports:
|
||||
- "8000:8000"
|
||||
|
||||
scheduler:
|
||||
build:
|
||||
context: ./yunke-server
|
||||
dockerfile: Dockerfile
|
||||
restart: unless-stopped
|
||||
# Scheduler entrypoint — same image, different command. Cadence is
|
||||
# configurable via env (FULL_SYNC_HOUR / MESSAGE_INTERVAL_HOURS) so
|
||||
# operators can tune without rebuilding.
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- app.sync.cli
|
||||
- scheduler
|
||||
depends_on:
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
DB_URL: mysql+asyncmy://${MYSQL_USER:-yunke}:${MYSQL_PASSWORD:-yunke}@mysql:3306/${MYSQL_DATABASE:-yunke}
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
MEDIA_DIR: /data/media
|
||||
YUNKE_BASE_URL: ${YUNKE_BASE_URL:-https://crm2.yunkecn.com}
|
||||
YUNKE_USERNAME: ${YUNKE_USERNAME:-}
|
||||
YUNKE_PASSWORD: ${YUNKE_PASSWORD:-}
|
||||
YK_USERID: ${YK_USERID:-}
|
||||
YK_COMPANY: ${YK_COMPANY:-}
|
||||
YK_COOKIE: ${YK_COOKIE:-}
|
||||
ENCRYPTION_KEY: ${ENCRYPTION_KEY:-}
|
||||
# Scheduler cadence (env-driven; see app.sync.cli._cmd_scheduler).
|
||||
FULL_SYNC_HOUR: ${FULL_SYNC_HOUR:-2}
|
||||
MESSAGE_INTERVAL_HOURS: ${MESSAGE_INTERVAL_HOURS:-4}
|
||||
volumes:
|
||||
# Share the media volume with backend so both services see the same
|
||||
# downloaded blobs (read-only would be ideal but the scheduler is the
|
||||
# writer; mounting read-write here is the simplest correct config).
|
||||
- media_data:/data/media
|
||||
|
||||
# ----------------------------------------------------------------------- #
|
||||
# Frontend — built + served by nginx
|
||||
# ----------------------------------------------------------------------- #
|
||||
frontend:
|
||||
build:
|
||||
context: ./yunkes
|
||||
dockerfile: Dockerfile
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
- backend
|
||||
ports:
|
||||
# Map nginx's :80 to the host's :5173 so the same URL
|
||||
# (http://localhost:5173) works in dev (vite) and prod (nginx).
|
||||
- "5173:80"
|
||||
|
||||
volumes:
|
||||
mysql_data:
|
||||
redis_data:
|
||||
media_data:
|
||||
Reference in New Issue
Block a user