diff --git a/yunke-server/app/api/schemas.py b/yunke-server/app/api/schemas.py index 5a3c110..e3648f4 100644 --- a/yunke-server/app/api/schemas.py +++ b/yunke-server/app/api/schemas.py @@ -182,6 +182,13 @@ class GroupOut(BaseModel): member_count: int | None = None message_count: int | None = None last_chat_time: int | None = None + # 群主/建群补充列(2026-08-25 对齐云客微信群 tab;scraper/回填脚本提取) + owner_name: str | None = None + owner_wxid: str | None = None + owner_alias: str | None = None + owner_avatar: str | None = None + friend_count: int | None = None + create_time: int | None = None # 建群时间 ms epoch raw_json: dict[str, Any] | None = None diff --git a/yunke-server/app/db/models.py b/yunke-server/app/db/models.py index bca1141..13cbf5f 100644 --- a/yunke-server/app/db/models.py +++ b/yunke-server/app/db/models.py @@ -290,6 +290,16 @@ class ChatGroup(Base): last_chat_time: Mapped[int | None] = mapped_column(BigInteger, nullable=True) update_time: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + # 通讯录·微信群 tab 对齐云客的补充列（2026-08-25，从 raw_json 提取： + # ownerName/ownerWchatId/ownerAlias/ownerImage/friendCount/createTime； + # createTime 兼容 ms-int 与 'YYYY-MM-DD HH:mm:SS' 两种上游形态，归一为 ms） + owner_name: Mapped[str | None] = mapped_column(String(255), nullable=True) # 群主昵称(上游脱敏) + owner_wxid: Mapped[str | None] = mapped_column(String(128), nullable=True) + owner_alias: Mapped[str | None] = mapped_column(String(255), nullable=True) # 群主微信号 + owner_avatar: Mapped[str | None] = mapped_column(String(1024), nullable=True) + friend_count: Mapped[int | None] = mapped_column(Integer, nullable=True) # 群成员中员工好友数 + create_time: Mapped[int | None] = mapped_column(BigInteger, nullable=True) # 建群时间 ms epoch + version: Mapped[str | None] = mapped_column(String(64), nullable=True) raw_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True) fetched_at: Mapped[dt.datetime] = _fetched_at() diff --git a/yunke-server/app/scraper/wechat.py b/yunke-server/app/scraper/wechat.py index 9bc04f5..7ef1dfa 100644 --- a/yunke-server/app/scraper/wechat.py +++ b/yunke-server/app/scraper/wechat.py @@ -889,6 +889,17 @@ def _normalize_chat_group(record: dict[str, Any]) -> dict[str, Any]: Adds ``channel="wechat"`` and preserves the original record under ``raw_json``. ``None``-safe on every field: missing fields become ``None`` rather than raising, so a partial payload does not break the scrape. + + 2026-08-25 补充列（对齐云客 员工聊天窗口·微信群 tab，真机验证 raw_json 键）： + + * ``ownerName`` -> ``owner_name`` (群主昵称，上游脱敏) + * ``ownerWchatId`` -> ``owner_wxid`` (群主微信号，上游脱敏) + * ``ownerAlias`` -> ``owner_alias`` (群主微信号别名) + * ``ownerImage`` -> ``owner_avatar`` (群主头像) + * ``friendCount`` -> ``friend_count`` (群成员中员工好友数) + * ``createTime`` -> ``create_time`` (建群时间;实测 smoke.db 全量为 + ``'YYYY-MM-DD HH:mm:ss'`` 字符串,另兼容 ms-int/数字串,统一归一为 + ms-epoch int,复用 ``_parse_add_time`` 的双形态解析) """ return { "room_id": record.get("friendWxId"), @@ -908,6 +919,13 @@ def _normalize_chat_group(record: dict[str, Any]) -> dict[str, Any]: "add_members": None, "last_chat_time": record.get("lastChatTime"), "update_time": None, + # 群主/建群补充列(2026-08-25;raw_json 键 -> 同名列) + "owner_name": record.get("ownerName"), + "owner_wxid": record.get("ownerWchatId"), + "owner_alias": record.get("ownerAlias"), + "owner_avatar": record.get("ownerImage"), + "friend_count": _coerce_int_or_none(record.get("friendCount")), + "create_time": _parse_add_time(record.get("createTime")), "version": record.get("version"), "raw_json": record, } diff --git a/yunke-server/migrations/versions/099ac2097fa2_chat_group_owner_cols_and_composite_pk.py b/yunke-server/migrations/versions/099ac2097fa2_chat_group_owner_cols_and_composite_pk.py new file mode 100644 index 0000000..a42bed3 --- /dev/null +++ b/yunke-server/migrations/versions/099ac2097fa2_chat_group_owner_cols_and_composite_pk.py @@ -0,0 +1,108 @@ +"""chat_group 群主补充列 + 复合主键修复 + +Revision ID: 099ac2097fa2 +Revises: 524da8d2d597 +Create Date: 2026-08-25 14:26:06.532289 + +内容 +---- +1. 对齐云客 员工聊天窗口·微信群 tab 的 6 个补充列(2026-08-25 真机验证 + raw_json 键 ownerName/ownerWchatId/ownerAlias/ownerImage/friendCount/ + createTime,由 scraper 与 scripts/backfill_chat_group_fields.py 提取; + createTime 实测为 'YYYY-MM-DD HH:mm:ss' 字符串,归一为 ms-epoch): + owner_name / owner_wxid / owner_alias / owner_avatar / friend_count / + create_time。 +2. 修复模型-迁移 drift(Task 1 评审遗留):模型 PK 自 2026-08-24(commit + 8a99940)起为复合键 (room_id, channel, owner_account_id),但迁移链 + initial_schema 仍建单列 PrimaryKeyConstraint('room_id') 且 + owner_account_id nullable。本迁移把 PK 变更为复合键并将 + owner_account_id 置 NOT NULL(smoke.db 实测 1105 行无 NULL,可安全 + 变更;MySQL 存量若存在 NULL 行会在此报错——那是必须人工核对的数据 + 问题,不用兜底值编造)。 + +SQLite 不支持直接改 PK,batch_alter_table 自动走表重建;copy_from 显式 +给出迁移前表结构(含命名 PK 与索引)以便 drop/recreate 定位约束。 +MySQL 无需重建,渲染为直接 ALTER(ADD COLUMN / MODIFY / DROP PRIMARY +KEY / ADD PRIMARY KEY)。 +""" +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + + +# revision identifiers, used by Alembic. +revision: str = '099ac2097fa2' +down_revision: str | Sequence[str] | None = '524da8d2d597' +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +# 迁移前的 chat_group 结构(initial_schema 原样;PK 命名仅为 batch +# drop_constraint 定位用,实际库中为匿名约束)。 +_chat_group_before = sa.Table( + "chat_group", + sa.MetaData(), + sa.Column("room_id", sa.String(128), nullable=False), + sa.Column("channel", sa.String(16), nullable=False), + sa.Column("room_name", sa.String(255), nullable=True), + sa.Column("owner_account_id", sa.String(128), nullable=True), + sa.Column("owner_account_nickname", sa.String(255), nullable=True), + sa.Column("member_count", sa.Integer(), nullable=True), + sa.Column("member_ids", sa.Text(), nullable=True), + sa.Column("head_image_url", sa.String(1024), nullable=True), + sa.Column("owner_user_id", sa.String(64), nullable=True), + sa.Column("owner_user_phone", sa.String(64), nullable=True), + sa.Column("message_count", sa.Integer(), nullable=True), + sa.Column("own_message_count", sa.Integer(), nullable=True), + sa.Column("loss_members", sa.Integer(), nullable=True), + sa.Column("add_members", sa.Integer(), nullable=True), + sa.Column("last_chat_time", sa.BigInteger(), nullable=True), + sa.Column("update_time", sa.BigInteger(), nullable=True), + sa.Column("version", sa.String(64), nullable=True), + sa.Column("raw_json", sa.JSON(), nullable=True), + sa.Column( + "fetched_at", sa.DateTime(), server_default=sa.func.now(), nullable=False + ), + sa.Column( + "updated_at", sa.DateTime(), server_default=sa.func.now(), nullable=False + ), + sa.PrimaryKeyConstraint("room_id", name="pk_chat_group"), + sa.Index("ix_chat_group_owner", "channel", "owner_account_id", "room_id"), +) + + +def upgrade() -> None: + """Upgrade schema.""" + with op.batch_alter_table( + "chat_group", schema=None, copy_from=_chat_group_before + ) as batch_op: + # 1) 群主/建群补充列(全部 nullable,存量默认 NULL,回填脚本再提取) + batch_op.add_column(sa.Column("owner_name", sa.String(255), nullable=True)) + batch_op.add_column(sa.Column("owner_wxid", sa.String(128), nullable=True)) + batch_op.add_column(sa.Column("owner_alias", sa.String(255), nullable=True)) + batch_op.add_column(sa.Column("owner_avatar", sa.String(1024), nullable=True)) + batch_op.add_column(sa.Column("friend_count", sa.Integer(), nullable=True)) + batch_op.add_column(sa.Column("create_time", sa.BigInteger(), nullable=True)) + # 2) 复合 PK 对齐模型(2026-08-24 8a99940):单列 PK 会把多员工同群 + # 错误去重;owner_account_id 入键故需 NOT NULL。 + batch_op.alter_column( + "owner_account_id", existing_type=sa.String(128), nullable=False + ) + batch_op.drop_constraint("pk_chat_group", type_="primary") + batch_op.create_primary_key( + "pk_chat_group", ["room_id", "channel", "owner_account_id"] + ) + + +def downgrade() -> None: + """Downgrade schema.""" + # 降级需先把可能存在的多 owner 同群行去重才能退回单列 PK——不做自动 + # 删行,保留数据由人工处理;仅删列。 + with op.batch_alter_table("chat_group", schema=None) as batch_op: + batch_op.drop_column("owner_name") + batch_op.drop_column("owner_wxid") + batch_op.drop_column("owner_alias") + batch_op.drop_column("owner_avatar") + batch_op.drop_column("friend_count") + batch_op.drop_column("create_time") diff --git a/yunke-server/scripts/backfill_chat_group_fields.py b/yunke-server/scripts/backfill_chat_group_fields.py new file mode 100644 index 0000000..958d373 --- /dev/null +++ b/yunke-server/scripts/backfill_chat_group_fields.py @@ -0,0 +1,103 @@ +"""Backfill ChatGroup 群主/建群补充列 from raw_json (no re-scrape). + +2026-08-25 对齐云客 员工聊天窗口·微信群 tab 的 6 个补充列(迁移 +099ac2097fa2 加列):从既有行的 raw_json 提取 ownerName/ownerWchatId/ +ownerAlias/ownerImage/friendCount/createTime 写入同名列。 + +幂等:逐列只在「列为空 且 raw_json 有可用值」时回填,重跑只补新空列。 +createTime 兼容 ms-int 与 'YYYY-MM-DD HH:mm:ss'(smoke.db 实测全量为 +后者),复用 scraper 层 _parse_add_time 归一为 ms-epoch int。 +""" +import asyncio +import time + +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from app.db import models # noqa: F401 +from app.db.models import ChatGroup +from app.scraper.wechat import _coerce_int_or_none, _parse_add_time + +BATCH = 200 + +# raw_json 键 -> 目标列(字符串直提) +_STR_FIELDS: dict[str, str] = { + "ownerName": "owner_name", + "ownerWchatId": "owner_wxid", + "ownerAlias": "owner_alias", + "ownerImage": "owner_avatar", +} + + +def _extract(raw: dict, current: dict) -> dict: + """raw_json -> 列值(只为「当前为空」的列产出值)。""" + out: dict = {} + for key, col in _STR_FIELDS.items(): + if current.get(col) in (None, "") and raw.get(key): + out[col] = raw[key] + if current.get("friend_count") is None: + v = _coerce_int_or_none(raw.get("friendCount")) + if v is not None: + out["friend_count"] = v + if current.get("create_time") is None: + v = _parse_add_time(raw.get("createTime")) + if v is not None: + out["create_time"] = v + return out + + +async def main(db_url: str = "sqlite+aiosqlite:///./smoke.db") -> None: + engine = create_async_engine(db_url) + sm = async_sessionmaker(engine, expire_on_commit=False) + scanned = updated = 0 + async with sm() as s: + rows = ( + await s.execute( + select( + ChatGroup.room_id, + ChatGroup.channel, + ChatGroup.owner_account_id, + ChatGroup.raw_json, + ChatGroup.owner_name, + ChatGroup.owner_wxid, + ChatGroup.owner_alias, + ChatGroup.owner_avatar, + ChatGroup.friend_count, + ChatGroup.create_time, + ) + ) + ).all() + for rid, ch, ow, raw, *cols in rows: + scanned += 1 + if not raw: + continue + current = dict( + zip( + ("owner_name", "owner_wxid", "owner_alias", "owner_avatar", + "friend_count", "create_time"), + cols, + ) + ) + values = _extract(raw, current) + if not values: + continue + await s.execute( + update(ChatGroup) + .where( + ChatGroup.room_id == rid, + ChatGroup.channel == ch, + ChatGroup.owner_account_id == ow, + ) + .values(**values) + ) + updated += 1 + if updated % BATCH == 0: + await s.commit() + print(f"[{time.strftime('%H:%M:%S')}] {updated} updated...") + await s.commit() + print(f"backfilled {updated}/{scanned} rows") + await engine.dispose() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/yunke-server/tests/test_api_groups.py b/yunke-server/tests/test_api_groups.py index d54ed8d..e26e73b 100644 --- a/yunke-server/tests/test_api_groups.py +++ b/yunke-server/tests/test_api_groups.py @@ -255,3 +255,51 @@ async def test_room_id_filter_returns_single_group(make_client): body = r.json() assert body["total"] == 1 assert body["items"][0]["room_name"] == "群二" + + +# --------------------------------------------------------------------------- +# 群主/建群补充列（2026-08-25 对齐云客员工聊天窗口·微信群 tab） +# raw_json 键 ownerName/ownerWchatId/ownerAlias/ownerImage/friendCount/createTime +# 由 scraper(_normalize_chat_group) 与 scripts/backfill_chat_group_fields.py +# 提取进同名列;API 直接输出列值。 +# --------------------------------------------------------------------------- + +async def test_groups_owner_fields(make_client): + async def seed(session): + session.add(ChatGroup(room_id="123@chatroom", channel="wechat", room_name="测试群", + owner_account_id="wxid_x", member_count=10, + owner_name="Q**e", owner_wxid="RL3*****108", owner_alias="Fr******er", + owner_avatar="https://wx.qlogo.cn/o/0", friend_count=22, + create_time=1740000000000, + raw_json={"ownerName": "Q**e", "ownerWchatId": "RL3*****108", "ownerAlias": "Fr******er", + "ownerImage": "https://wx.qlogo.cn/o/0", "friendCount": 22, + "createTime": 1740000000000})) + await session.commit() + + client = await make_client(seed) + r = await client.get("/api/groups", params={"room_id": "123@chatroom"}) + assert r.status_code == 200 + item = r.json()["items"][0] + assert item["owner_name"] == "Q**e" + assert item["owner_wxid"] == "RL3*****108" + assert item["owner_alias"] == "Fr******er" + assert item["owner_avatar"] == "https://wx.qlogo.cn/o/0" + assert item["friend_count"] == 22 + assert item["create_time"] == 1740000000000 + + +async def test_groups_owner_fields_default_null(make_client): + """未回填/上游缺失的行走 schema 默认值 null(不报错、不缺 key)。""" + + async def seed(session): + session.add(ChatGroup(room_id="old@chatroom", channel="wechat", + owner_account_id="wxid_y")) + await session.commit() + + client = await make_client(seed) + r = await client.get("/api/groups", params={"room_id": "old@chatroom"}) + assert r.status_code == 200 + item = r.json()["items"][0] + for key in ("owner_name", "owner_wxid", "owner_alias", + "owner_avatar", "friend_count", "create_time"): + assert item[key] is None diff --git a/yunke-server/tests/test_scrape_wechat_groups.py b/yunke-server/tests/test_scrape_wechat_groups.py index e19c7ff..4091675 100644 --- a/yunke-server/tests/test_scrape_wechat_groups.py +++ b/yunke-server/tests/test_scrape_wechat_groups.py @@ -32,7 +32,7 @@ from typing import Any import httpx import respx -from app.scraper.wechat import scrape_wechat_groups +from app.scraper.wechat import _normalize_chat_group, scrape_wechat_groups from app.session.yunke_client import YunkeClient from tests.cassettes.wxFriends_fixture import make_friend_list_response @@ -277,3 +277,50 @@ async def test_scrape_wechat_groups_stops_on_empty_first_page(): assert results == [] assert route.call_count == 1 + + +# --------------------------------------------------------------------------- +# 群主/建群补充列提取(2026-08-25 对齐云客员工聊天窗口·微信群 tab) +# raw_json 键: ownerName/ownerWchatId/ownerAlias/ownerImage/friendCount/createTime +# smoke.db 实测: createTime 全部为 'YYYY-MM-DD HH:mm:ss' 字符串;ms int 直通。 +# --------------------------------------------------------------------------- + + +def test_normalize_chat_group_extracts_owner_fields(): + """6 个补充键提取到同名列;date-str createTime 归一为 ms epoch。""" + record = { + "friendWxId": "123@chatroom", + "friendWxNickname": "测试群", + "friendNum": 25, + "wxId": "wxid_x", + "ownerName": "Q**e", + "ownerWchatId": "RL3*****108", + "ownerAlias": "Fr******er", + "ownerImage": "https://wx.qlogo.cn/o/0", + "friendCount": 22, + "createTime": "2025-10-27 13:30:55", + } + out = _normalize_chat_group(record) + assert out["owner_name"] == "Q**e" + assert out["owner_wxid"] == "RL3*****108" + assert out["owner_alias"] == "Fr******er" + assert out["owner_avatar"] == "https://wx.qlogo.cn/o/0" + assert out["friend_count"] == 22 + # 'YYYY-MM-DD HH:mm:ss' -> ms epoch(naive UTC,同 addTime 约定) + assert out["create_time"] == 1761571855000 + + +def test_normalize_chat_group_owner_fields_none_safe(): + """缺失全部 6 个键 -> 全 None(老 capture 不炸)。""" + out = _normalize_chat_group({"friendWxId": "r@chatroom"}) + for key in ("owner_name", "owner_wxid", "owner_alias", + "owner_avatar", "friend_count", "create_time"): + assert out[key] is None + + +def test_normalize_chat_group_create_time_ms_passthrough(): + """ms-epoch int 直通(不做日期解析)。""" + out = _normalize_chat_group( + {"friendWxId": "r@chatroom", "createTime": 1740000000000} + ) + assert out["create_time"] == 1740000000000