diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index ee64b5f..539f51b 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -74,14 +74,53 @@ docker compose ps # 五服务全部 Up(healthy) docker compose logs -f scheduler # 看调度器日志 ``` -### 4. 验证 +### 4. 数据库迁移（alembic schema） + +后端启动**不会**自动建表，schema 由 alembic 迁移链管理（镜像已打包 +`alembic.ini` + `migrations/`，`DB_URL` 由 compose 注入）。 + +**新库首次部署**（空 MySQL 直接建全量表）： + +```bash +docker compose exec backend alembic upgrade head +``` + +**旧库升级**（2026-08-22 `migrate_sqlite_to_mysql.py` 搬迁产物，没有 +alembic_version 表）——2026-08-25 实战踩通的完整路径，原样记录： + +```bash +# 1) 前置校验:chat_group 无 NULL owner_account_id(复合 PK 迁移要求 NOT NULL, +# 非 0 则是必须人工核对的数据问题,不要带病升级) +docker compose exec mysql mysql -uyunke -p<密码> yunke \ + -e "SELECT COUNT(*) FROM chat_group WHERE owner_account_id IS NULL" + +# 2) stamp 到搬迁脚本对应的链上位置,再升级到 head +docker compose exec backend alembic stamp d8eee96cd017 +docker compose exec backend alembic upgrade head +``` + +**每次拉新代码后**： + +```bash +docker compose exec backend alembic upgrade head +``` + +涉及存量数据回填的更新另跑回填脚本（幂等，重跑 0 改动；脚本默认连 +`./smoke.db`，服务器上须显式传 DB_URL——容器 env 已带，直接透传）： + +```bash +docker compose exec backend python -c "import asyncio,os; from scripts.backfill_chat_group_fields import main; asyncio.run(main(os.environ['DB_URL']))" +docker compose exec backend python -c "import asyncio,os; from scripts.backfill_msg_types import main; asyncio.run(main(os.environ['DB_URL']))" +``` + +### 5. 验证 ```bash curl http://localhost:8000/api/health # {"status":"ok"} curl http://localhost:5173/ # 前端页面 ``` -### 5. 数据迁移（把本地 SQLite 数据搬到服务器 MySQL） +### 6. 数据迁移（把本地 SQLite 数据搬到服务器 MySQL） **方式 A：重抓（推荐，数据最新）** @@ -100,7 +139,7 @@ DB_URL="mysql+asyncmy://yunke:<密码>@mysql:3306/yunke" \ 把本地 smoke.db 和 media/ 目录传到服务器，用 Python 脚本逐表导： （如有需要我可以写迁移脚本，~100 行） -### 6. 媒体文件 +### 7. 媒体文件 媒体存在 Docker volume `media_data` 里。本地已有 2.5GB 媒体的话： @@ -122,7 +161,7 @@ docker run --rm -v yunke_media_data:/target -v $(pwd):/src alpine \ | 换 cookie | 改 `.env` 的 `YK_COOKIE` → `docker compose restart scheduler backend` | | 备份 MySQL | `docker compose exec mysql mysqldump -uroot -p<密码> yunke > backup.sql` | | 备份媒体 | `docker run --rm -v yunke_media_data:/data -v $(pwd):/out alpine tar czf /out/media.tgz -C /data .` | -| 升级版本 | `git pull && docker compose up -d --build` | +| 升级版本 | `git pull && docker compose up -d --build && docker compose exec backend alembic upgrade head`（回填见「数据库迁移」） | ## 已知注意事项 diff --git a/yunke-server/Dockerfile b/yunke-server/Dockerfile index c30cc65..ca52c5f 100644 --- a/yunke-server/Dockerfile +++ b/yunke-server/Dockerfile @@ -51,6 +51,12 @@ COPY app/ ./app/ # full_sync_all.sh 由 scheduler._page_subprocess 以 cwd=/app 调起, # 2026-08-21 部署阻塞点#2:镜像里必须带上)。 COPY scripts/ ./scripts/ +# alembic.ini + migrations/ — 容器内可跑 `alembic upgrade head`(DB_URL +# 环境变量已由 compose 注入;entrypoint 不自动迁移,由运维手动执行, +# 见 docs/DEPLOY.md「数据库迁移」)。alembic.ini 的 script_location 用 +# %(here)s 相对 ini 自身,落在 /app/migrations 正好对上。 +COPY alembic.ini ./ +COPY migrations/ ./migrations/ # Install the package + dev dependencies (the dev extras include aiosqlite + # pytest, which are tiny and convenient for in-container smoke checks). Pin diff --git a/yunke-server/app/api/routers/groups.py b/yunke-server/app/api/routers/groups.py index 2ac8767..a909cf1 100644 --- a/yunke-server/app/api/routers/groups.py +++ b/yunke-server/app/api/routers/groups.py @@ -44,9 +44,9 @@ async def list_groups( ``user_id`` maps to the ``ChatGroup.owner_user_id`` column (the staff ``userId`` who owns the chatroom account), mirroring the ``user_id`` - filter on ``GET /api/customers``. ``room_id`` is an exact-match single- - row lookup (chat page group-name resolution). ``q`` is a substring match - on ``room_name``. + filter on ``GET /api/customers``. ``room_id`` is an exact-match lookup + (composite PK: the same room may return one row per owner; chat page + group-name resolution). ``q`` is a substring match on ``room_name``. """ conditions = [] if channel is not None: diff --git a/yunke-server/app/api/routers/messages.py b/yunke-server/app/api/routers/messages.py index 27cfdf5..43cfb96 100644 --- a/yunke-server/app/api/routers/messages.py +++ b/yunke-server/app/api/routers/messages.py @@ -52,19 +52,9 @@ _MSG_TYPE_FILTER: dict[str, tuple[str, ...]] = { "mini_program": ("mini_program",), } -MsgTypeParam = Literal[ - "text", - "image", - "voice", - "video", - "link", - "file", - "red_packet", - "transfer", - "voice_call", - "video_call", - "mini_program", -] +# 派生而非重复罗列:加一类只改上面的 dict,Literal 与 422 校验自动跟上 +# (两处手写曾会漂移——漏一边即 500/422 不一致) +MsgTypeParam = Literal[tuple(_MSG_TYPE_FILTER)] @router.get("/messages") diff --git a/yunke-server/app/db/models.py b/yunke-server/app/db/models.py index 13cbf5f..f8afebb 100644 --- a/yunke-server/app/db/models.py +++ b/yunke-server/app/db/models.py @@ -262,9 +262,11 @@ class Customer(Base): class ChatGroup(Base): """聊天群 — source: ``POST /wechatweb/chatroom/getChatroomList``. - PK ``room_id`` is the source ``chatRoomId`` (ends with ``@chatroom``). - The raw ``memberIds`` (semicolon-joined wxids) is kept verbatim in - ``member_ids``; the parsed members are mirrored into :class:`GroupMember`. + PK is composite (``room_id``, ``channel``, ``owner_account_id``) — the + same room mirrors to one row per owning staff account; ``room_id`` is the + source ``chatRoomId`` (ends with ``@chatroom``). The raw ``memberIds`` + (semicolon-joined wxids) is kept verbatim in ``member_ids``; the parsed + members are mirrored into :class:`GroupMember`. """ __tablename__ = "chat_group" diff --git a/yunke-server/migrations/versions/9b0d0c133035_wechat_account_list_order.py b/yunke-server/migrations/versions/9b0d0c133035_wechat_account_list_order.py new file mode 100644 index 0000000..32eb729 --- /dev/null +++ b/yunke-server/migrations/versions/9b0d0c133035_wechat_account_list_order.py @@ -0,0 +1,48 @@ +"""wechat_account 补 list_order 列 + +Revision ID: 9b0d0c133035 +Revises: 099ac2097fa2 +Create Date: 2026-08-25 20:16:46.324537 + +内容 +---- +补齐模型-迁移 drift:``wechat_account.list_order``(云客 wxUser/list 返回 +顺序,2026-08-24 加进模型)一直没有对应迁移——干净库(create_all 或纯 +alembic 链)上模型建表含该列、迁移链建表不含,``/api/accounts`` SELECT +即 500(实测两次,Task 11/12 验证时被迫对 smoke.db 手动 ALTER)。 + +幂等:smoke.db 已被手动加过该列(其 alembic_version 仍指旧 head),直接 +add_column 会报 duplicate column——用 ``sa.inspect`` 先查列是否存在, +存在则跳过(SQLite/MySQL 通用,不需要裸 PRAGMA/information_schema)。 +downgrade 对称:列不存在时跳过。 +""" +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + + +# revision identifiers, used by Alembic. +revision: str = '9b0d0c133035' +down_revision: str | Sequence[str] | None = '099ac2097fa2' +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _has_column(table: str, column: str) -> bool: + insp = sa.inspect(op.get_bind()) + return column in {c["name"] for c in insp.get_columns(table)} + + +def upgrade() -> None: + """Upgrade schema.""" + if _has_column("wechat_account", "list_order"): + return # 已手动加过(smoke.db),幂等跳过 + op.add_column("wechat_account", sa.Column("list_order", sa.Integer(), nullable=True)) + + +def downgrade() -> None: + """Downgrade schema.""" + if not _has_column("wechat_account", "list_order"): + return + op.drop_column("wechat_account", "list_order") diff --git a/yunke-server/migrations/versions/fe63c6789e94_align_task14_model_type_fixes.py b/yunke-server/migrations/versions/fe63c6789e94_align_task14_model_type_fixes.py new file mode 100644 index 0000000..bc84cc9 --- /dev/null +++ b/yunke-server/migrations/versions/fe63c6789e94_align_task14_model_type_fixes.py @@ -0,0 +1,115 @@ +"""Task 14 模型类型修正落进迁移链(URL Text + db_size BigInteger) + +Revision ID: fe63c6789e94 +Revises: 9b0d0c133035 +Create Date: 2026-08-25 20:34:26.646844 + +内容 +---- +补齐模型-迁移 drift:2026-08-22 部署实战(commit 00ae13b)在**模型**上改了 +5 列类型,但迁移链从未跟上—— + +* message.file_url / file_th_url / file_hd_url、media.original_url: + String(1024) → Text(分享链接可超 1KB,MySQL VARCHAR(1024) 会截断); +* wechat_account.db_size: Integer → BigInteger(字节数可超 32 位 INT, + 实测 2.17GB)。 + +后果:纯 alembic 链建的干净库(deploy 文档的新库路径)拿到旧类型,静默 +复活已修掉的截断问题;``alembic check`` 常红。本迁移把链对齐到模型。 + +幂等性:ALTER MODIFY 到相同类型是合法 SQL——MySQL 上 Docker 库的表由 +migrate_sqlite_to_mysql(按当时模型 create_all,已是 Text/BIGINT)创建, +重放本迁移为无害 no-op,无需条件守卫。SQLite 走 batch_alter_table 表重建 +(仅改声明类型,TEXT/INTEGER 亲和性不变,数据原样拷贝)。 + +⚠️ SQLite 侧副作用:message 表重建会丢其上的 FTS5 同步触发器、且 +external-content 索引(message_fts 按 rowid 映射)会与新表 rowid 脱节。 +MySQL(prod)无 FTS 不受影响;SQLite dev 库跑完本迁移后需执行 +``ensure_fts``(重建触发器)+ ``rebuild_fts``(按新 rowid 重灌索引, +app/db/fts.py 现成函数)修复搜索。 +""" +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + + +# revision identifiers, used by Alembic. +revision: str = 'fe63c6789e94' +down_revision: str | Sequence[str] | None = '9b0d0c133035' +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Upgrade schema.""" + with op.batch_alter_table("message") as batch_op: + batch_op.alter_column( + "file_url", + existing_type=sa.String(1024), + type_=sa.Text(), + existing_nullable=True, + ) + batch_op.alter_column( + "file_th_url", + existing_type=sa.String(1024), + type_=sa.Text(), + existing_nullable=True, + ) + batch_op.alter_column( + "file_hd_url", + existing_type=sa.String(1024), + type_=sa.Text(), + existing_nullable=True, + ) + with op.batch_alter_table("media") as batch_op: + batch_op.alter_column( + "original_url", + existing_type=sa.String(1024), + type_=sa.Text(), + existing_nullable=True, + ) + with op.batch_alter_table("wechat_account") as batch_op: + batch_op.alter_column( + "db_size", + existing_type=sa.Integer(), + type_=sa.BigInteger(), + existing_nullable=True, + ) + + +def downgrade() -> None: + """Downgrade schema.""" + with op.batch_alter_table("wechat_account") as batch_op: + batch_op.alter_column( + "db_size", + existing_type=sa.BigInteger(), + type_=sa.Integer(), + existing_nullable=True, + ) + with op.batch_alter_table("media") as batch_op: + batch_op.alter_column( + "original_url", + existing_type=sa.Text(), + type_=sa.String(1024), + existing_nullable=True, + ) + with op.batch_alter_table("message") as batch_op: + batch_op.alter_column( + "file_hd_url", + existing_type=sa.Text(), + type_=sa.String(1024), + existing_nullable=True, + ) + batch_op.alter_column( + "file_th_url", + existing_type=sa.Text(), + type_=sa.String(1024), + existing_nullable=True, + ) + batch_op.alter_column( + "file_url", + existing_type=sa.Text(), + type_=sa.String(1024), + existing_nullable=True, + ) diff --git a/yunke-server/scripts/scrape_via_page.py b/yunke-server/scripts/scrape_via_page.py index 3fa4228..d4ac919 100644 --- a/yunke-server/scripts/scrape_via_page.py +++ b/yunke-server/scripts/scrape_via_page.py @@ -1561,6 +1561,8 @@ async def _iter_moments_pages( rows = data.get("data") if isinstance(data, dict) else None if not isinstance(rows, list): break + # 每页一行进度(镜像 _fetch_pages 的 " page N: X rows") + logger.info(f" friendster page {page_num}: {len(rows)} rows") for row in rows: if not isinstance(row, dict): continue diff --git a/yunke-server/tests/test_api_groups.py b/yunke-server/tests/test_api_groups.py index 6dfb4cb..616cbe9 100644 --- a/yunke-server/tests/test_api_groups.py +++ b/yunke-server/tests/test_api_groups.py @@ -241,7 +241,7 @@ async def test_members_empty_room_is_200(make_client): # room_id 精确过滤（2026-08-20：群聊页群名单行直查,替代超上限的大页拉取） # --------------------------------------------------------------------------- -async def test_room_id_filter_returns_single_group(make_client): +async def test_room_id_filter_returns_matching_groups(make_client): async def seed(session): session.add_all([ ChatGroup(room_id="r1@chatroom", channel="wechat", room_name="群一", owner_account_id="o1"),