feat(groups): chat_group 补 6 群主/建群列 + 复合PK迁移修复 + 存量回填
对齐云客员工聊天窗口·微信群 tab(2026-08-25 真机验证 raw_json 键): - ChatGroup 加列 owner_name/owner_wxid/owner_alias/owner_avatar/ friend_count/create_time(全 nullable) - scraper _normalize_chat_group 同名提取;createTime 兼容 ms-int 与 'YYYY-MM-DD HH:mm:ss'(smoke.db 实测 1105 行全为后者),复用 _parse_add_time 归一 ms-epoch - 迁移 099ac2097fa2:加 6 列 + 修复 Task1 评审遗留 drift——PK 由单列 room_id 变更为复合 (room_id, channel, owner_account_id) 且 owner_account_id NOT NULL(对齐模型 8a99940;smoke.db 1105 行无 NULL, 未兜底编造数据);SQLite 走 batch 重建(copy_from 显式旧结构), MySQL 渲染为直接 ALTER(离线 --sql 已验证) - 回填脚本 scripts/backfill_chat_group_fields.py(幂等,逐列只补空); smoke.db 实跑 backfilled 1105/1105,重跑 0/1105 - GroupOut 加 6 字段(model_validate 自动输出,路由无改) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -182,6 +182,13 @@ class GroupOut(BaseModel):
|
||||
member_count: int | None = None
|
||||
message_count: int | None = None
|
||||
last_chat_time: int | None = None
|
||||
# 群主/建群补充列(2026-08-25 对齐云客微信群 tab;scraper/回填脚本提取)
|
||||
owner_name: str | None = None
|
||||
owner_wxid: str | None = None
|
||||
owner_alias: str | None = None
|
||||
owner_avatar: str | None = None
|
||||
friend_count: int | None = None
|
||||
create_time: int | None = None # 建群时间 ms epoch
|
||||
raw_json: dict[str, Any] | None = None
|
||||
|
||||
|
||||
|
||||
@@ -290,6 +290,16 @@ class ChatGroup(Base):
|
||||
last_chat_time: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
||||
update_time: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
||||
|
||||
# 通讯录·微信群 tab 对齐云客的补充列（2026-08-25，从 raw_json 提取：
|
||||
# ownerName/ownerWchatId/ownerAlias/ownerImage/friendCount/createTime；
|
||||
# createTime 兼容 ms-int 与 'YYYY-MM-DD HH:mm:SS' 两种上游形态，归一为 ms）
|
||||
owner_name: Mapped[str | None] = mapped_column(String(255), nullable=True) # 群主昵称(上游脱敏)
|
||||
owner_wxid: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
owner_alias: Mapped[str | None] = mapped_column(String(255), nullable=True) # 群主微信号
|
||||
owner_avatar: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
friend_count: Mapped[int | None] = mapped_column(Integer, nullable=True) # 群成员中员工好友数
|
||||
create_time: Mapped[int | None] = mapped_column(BigInteger, nullable=True) # 建群时间 ms epoch
|
||||
|
||||
version: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
raw_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
||||
fetched_at: Mapped[dt.datetime] = _fetched_at()
|
||||
|
||||
@@ -889,6 +889,17 @@ def _normalize_chat_group(record: dict[str, Any]) -> dict[str, Any]:
|
||||
Adds ``channel="wechat"`` and preserves the original record under
|
||||
``raw_json``. ``None``-safe on every field: missing fields become ``None``
|
||||
rather than raising, so a partial payload does not break the scrape.
|
||||
|
||||
2026-08-25 补充列（对齐云客 员工聊天窗口·微信群 tab，真机验证 raw_json 键）：
|
||||
|
||||
* ``ownerName`` -> ``owner_name`` (群主昵称，上游脱敏)
|
||||
* ``ownerWchatId`` -> ``owner_wxid`` (群主微信号，上游脱敏)
|
||||
* ``ownerAlias`` -> ``owner_alias`` (群主微信号别名)
|
||||
* ``ownerImage`` -> ``owner_avatar`` (群主头像)
|
||||
* ``friendCount`` -> ``friend_count`` (群成员中员工好友数)
|
||||
* ``createTime`` -> ``create_time`` (建群时间;实测 smoke.db 全量为
|
||||
``'YYYY-MM-DD HH:mm:ss'`` 字符串,另兼容 ms-int/数字串,统一归一为
|
||||
ms-epoch int,复用 ``_parse_add_time`` 的双形态解析)
|
||||
"""
|
||||
return {
|
||||
"room_id": record.get("friendWxId"),
|
||||
@@ -908,6 +919,13 @@ def _normalize_chat_group(record: dict[str, Any]) -> dict[str, Any]:
|
||||
"add_members": None,
|
||||
"last_chat_time": record.get("lastChatTime"),
|
||||
"update_time": None,
|
||||
# 群主/建群补充列(2026-08-25;raw_json 键 -> 同名列)
|
||||
"owner_name": record.get("ownerName"),
|
||||
"owner_wxid": record.get("ownerWchatId"),
|
||||
"owner_alias": record.get("ownerAlias"),
|
||||
"owner_avatar": record.get("ownerImage"),
|
||||
"friend_count": _coerce_int_or_none(record.get("friendCount")),
|
||||
"create_time": _parse_add_time(record.get("createTime")),
|
||||
"version": record.get("version"),
|
||||
"raw_json": record,
|
||||
}
|
||||
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
"""chat_group 群主补充列 + 复合主键修复
|
||||
|
||||
Revision ID: 099ac2097fa2
|
||||
Revises: 524da8d2d597
|
||||
Create Date: 2026-08-25 14:26:06.532289
|
||||
|
||||
内容
|
||||
----
|
||||
1. 对齐云客 员工聊天窗口·微信群 tab 的 6 个补充列(2026-08-25 真机验证
|
||||
raw_json 键 ownerName/ownerWchatId/ownerAlias/ownerImage/friendCount/
|
||||
createTime,由 scraper 与 scripts/backfill_chat_group_fields.py 提取;
|
||||
createTime 实测为 'YYYY-MM-DD HH:mm:ss' 字符串,归一为 ms-epoch):
|
||||
owner_name / owner_wxid / owner_alias / owner_avatar / friend_count /
|
||||
create_time。
|
||||
2. 修复模型-迁移 drift(Task 1 评审遗留):模型 PK 自 2026-08-24(commit
|
||||
8a99940)起为复合键 (room_id, channel, owner_account_id),但迁移链
|
||||
initial_schema 仍建单列 PrimaryKeyConstraint('room_id') 且
|
||||
owner_account_id nullable。本迁移把 PK 变更为复合键并将
|
||||
owner_account_id 置 NOT NULL(smoke.db 实测 1105 行无 NULL,可安全
|
||||
变更;MySQL 存量若存在 NULL 行会在此报错——那是必须人工核对的数据
|
||||
问题,不用兜底值编造)。
|
||||
|
||||
SQLite 不支持直接改 PK,batch_alter_table 自动走表重建;copy_from 显式
|
||||
给出迁移前表结构(含命名 PK 与索引)以便 drop/recreate 定位约束。
|
||||
MySQL 无需重建,渲染为直接 ALTER(ADD COLUMN / MODIFY / DROP PRIMARY
|
||||
KEY / ADD PRIMARY KEY)。
|
||||
"""
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '099ac2097fa2'
|
||||
down_revision: str | Sequence[str] | None = '524da8d2d597'
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
# 迁移前的 chat_group 结构(initial_schema 原样;PK 命名仅为 batch
|
||||
# drop_constraint 定位用,实际库中为匿名约束)。
|
||||
_chat_group_before = sa.Table(
|
||||
"chat_group",
|
||||
sa.MetaData(),
|
||||
sa.Column("room_id", sa.String(128), nullable=False),
|
||||
sa.Column("channel", sa.String(16), nullable=False),
|
||||
sa.Column("room_name", sa.String(255), nullable=True),
|
||||
sa.Column("owner_account_id", sa.String(128), nullable=True),
|
||||
sa.Column("owner_account_nickname", sa.String(255), nullable=True),
|
||||
sa.Column("member_count", sa.Integer(), nullable=True),
|
||||
sa.Column("member_ids", sa.Text(), nullable=True),
|
||||
sa.Column("head_image_url", sa.String(1024), nullable=True),
|
||||
sa.Column("owner_user_id", sa.String(64), nullable=True),
|
||||
sa.Column("owner_user_phone", sa.String(64), nullable=True),
|
||||
sa.Column("message_count", sa.Integer(), nullable=True),
|
||||
sa.Column("own_message_count", sa.Integer(), nullable=True),
|
||||
sa.Column("loss_members", sa.Integer(), nullable=True),
|
||||
sa.Column("add_members", sa.Integer(), nullable=True),
|
||||
sa.Column("last_chat_time", sa.BigInteger(), nullable=True),
|
||||
sa.Column("update_time", sa.BigInteger(), nullable=True),
|
||||
sa.Column("version", sa.String(64), nullable=True),
|
||||
sa.Column("raw_json", sa.JSON(), nullable=True),
|
||||
sa.Column(
|
||||
"fetched_at", sa.DateTime(), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at", sa.DateTime(), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
sa.PrimaryKeyConstraint("room_id", name="pk_chat_group"),
|
||||
sa.Index("ix_chat_group_owner", "channel", "owner_account_id", "room_id"),
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
with op.batch_alter_table(
|
||||
"chat_group", schema=None, copy_from=_chat_group_before
|
||||
) as batch_op:
|
||||
# 1) 群主/建群补充列(全部 nullable,存量默认 NULL,回填脚本再提取)
|
||||
batch_op.add_column(sa.Column("owner_name", sa.String(255), nullable=True))
|
||||
batch_op.add_column(sa.Column("owner_wxid", sa.String(128), nullable=True))
|
||||
batch_op.add_column(sa.Column("owner_alias", sa.String(255), nullable=True))
|
||||
batch_op.add_column(sa.Column("owner_avatar", sa.String(1024), nullable=True))
|
||||
batch_op.add_column(sa.Column("friend_count", sa.Integer(), nullable=True))
|
||||
batch_op.add_column(sa.Column("create_time", sa.BigInteger(), nullable=True))
|
||||
# 2) 复合 PK 对齐模型(2026-08-24 8a99940):单列 PK 会把多员工同群
|
||||
# 错误去重;owner_account_id 入键故需 NOT NULL。
|
||||
batch_op.alter_column(
|
||||
"owner_account_id", existing_type=sa.String(128), nullable=False
|
||||
)
|
||||
batch_op.drop_constraint("pk_chat_group", type_="primary")
|
||||
batch_op.create_primary_key(
|
||||
"pk_chat_group", ["room_id", "channel", "owner_account_id"]
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
# 降级需先把可能存在的多 owner 同群行去重才能退回单列 PK——不做自动
|
||||
# 删行,保留数据由人工处理;仅删列。
|
||||
with op.batch_alter_table("chat_group", schema=None) as batch_op:
|
||||
batch_op.drop_column("owner_name")
|
||||
batch_op.drop_column("owner_wxid")
|
||||
batch_op.drop_column("owner_alias")
|
||||
batch_op.drop_column("owner_avatar")
|
||||
batch_op.drop_column("friend_count")
|
||||
batch_op.drop_column("create_time")
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Backfill ChatGroup 群主/建群补充列 from raw_json (no re-scrape).
|
||||
|
||||
2026-08-25 对齐云客 员工聊天窗口·微信群 tab 的 6 个补充列(迁移
|
||||
099ac2097fa2 加列):从既有行的 raw_json 提取 ownerName/ownerWchatId/
|
||||
ownerAlias/ownerImage/friendCount/createTime 写入同名列。
|
||||
|
||||
幂等:逐列只在「列为空 且 raw_json 有可用值」时回填,重跑只补新空列。
|
||||
createTime 兼容 ms-int 与 'YYYY-MM-DD HH:mm:ss'(smoke.db 实测全量为
|
||||
后者),复用 scraper 层 _parse_add_time 归一为 ms-epoch int。
|
||||
"""
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.db import models # noqa: F401
|
||||
from app.db.models import ChatGroup
|
||||
from app.scraper.wechat import _coerce_int_or_none, _parse_add_time
|
||||
|
||||
BATCH = 200
|
||||
|
||||
# raw_json 键 -> 目标列(字符串直提)
|
||||
_STR_FIELDS: dict[str, str] = {
|
||||
"ownerName": "owner_name",
|
||||
"ownerWchatId": "owner_wxid",
|
||||
"ownerAlias": "owner_alias",
|
||||
"ownerImage": "owner_avatar",
|
||||
}
|
||||
|
||||
|
||||
def _extract(raw: dict, current: dict) -> dict:
|
||||
"""raw_json -> 列值(只为「当前为空」的列产出值)。"""
|
||||
out: dict = {}
|
||||
for key, col in _STR_FIELDS.items():
|
||||
if current.get(col) in (None, "") and raw.get(key):
|
||||
out[col] = raw[key]
|
||||
if current.get("friend_count") is None:
|
||||
v = _coerce_int_or_none(raw.get("friendCount"))
|
||||
if v is not None:
|
||||
out["friend_count"] = v
|
||||
if current.get("create_time") is None:
|
||||
v = _parse_add_time(raw.get("createTime"))
|
||||
if v is not None:
|
||||
out["create_time"] = v
|
||||
return out
|
||||
|
||||
|
||||
async def main(db_url: str = "sqlite+aiosqlite:///./smoke.db") -> None:
|
||||
engine = create_async_engine(db_url)
|
||||
sm = async_sessionmaker(engine, expire_on_commit=False)
|
||||
scanned = updated = 0
|
||||
async with sm() as s:
|
||||
rows = (
|
||||
await s.execute(
|
||||
select(
|
||||
ChatGroup.room_id,
|
||||
ChatGroup.channel,
|
||||
ChatGroup.owner_account_id,
|
||||
ChatGroup.raw_json,
|
||||
ChatGroup.owner_name,
|
||||
ChatGroup.owner_wxid,
|
||||
ChatGroup.owner_alias,
|
||||
ChatGroup.owner_avatar,
|
||||
ChatGroup.friend_count,
|
||||
ChatGroup.create_time,
|
||||
)
|
||||
)
|
||||
).all()
|
||||
for rid, ch, ow, raw, *cols in rows:
|
||||
scanned += 1
|
||||
if not raw:
|
||||
continue
|
||||
current = dict(
|
||||
zip(
|
||||
("owner_name", "owner_wxid", "owner_alias", "owner_avatar",
|
||||
"friend_count", "create_time"),
|
||||
cols,
|
||||
)
|
||||
)
|
||||
values = _extract(raw, current)
|
||||
if not values:
|
||||
continue
|
||||
await s.execute(
|
||||
update(ChatGroup)
|
||||
.where(
|
||||
ChatGroup.room_id == rid,
|
||||
ChatGroup.channel == ch,
|
||||
ChatGroup.owner_account_id == ow,
|
||||
)
|
||||
.values(**values)
|
||||
)
|
||||
updated += 1
|
||||
if updated % BATCH == 0:
|
||||
await s.commit()
|
||||
print(f"[{time.strftime('%H:%M:%S')}] {updated} updated...")
|
||||
await s.commit()
|
||||
print(f"backfilled {updated}/{scanned} rows")
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -255,3 +255,51 @@ async def test_room_id_filter_returns_single_group(make_client):
|
||||
body = r.json()
|
||||
assert body["total"] == 1
|
||||
assert body["items"][0]["room_name"] == "群二"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 群主/建群补充列（2026-08-25 对齐云客员工聊天窗口·微信群 tab）
|
||||
# raw_json 键 ownerName/ownerWchatId/ownerAlias/ownerImage/friendCount/createTime
|
||||
# 由 scraper(_normalize_chat_group) 与 scripts/backfill_chat_group_fields.py
|
||||
# 提取进同名列;API 直接输出列值。
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def test_groups_owner_fields(make_client):
|
||||
async def seed(session):
|
||||
session.add(ChatGroup(room_id="123@chatroom", channel="wechat", room_name="测试群",
|
||||
owner_account_id="wxid_x", member_count=10,
|
||||
owner_name="Q**e", owner_wxid="RL3*****108", owner_alias="Fr******er",
|
||||
owner_avatar="https://wx.qlogo.cn/o/0", friend_count=22,
|
||||
create_time=1740000000000,
|
||||
raw_json={"ownerName": "Q**e", "ownerWchatId": "RL3*****108", "ownerAlias": "Fr******er",
|
||||
"ownerImage": "https://wx.qlogo.cn/o/0", "friendCount": 22,
|
||||
"createTime": 1740000000000}))
|
||||
await session.commit()
|
||||
|
||||
client = await make_client(seed)
|
||||
r = await client.get("/api/groups", params={"room_id": "123@chatroom"})
|
||||
assert r.status_code == 200
|
||||
item = r.json()["items"][0]
|
||||
assert item["owner_name"] == "Q**e"
|
||||
assert item["owner_wxid"] == "RL3*****108"
|
||||
assert item["owner_alias"] == "Fr******er"
|
||||
assert item["owner_avatar"] == "https://wx.qlogo.cn/o/0"
|
||||
assert item["friend_count"] == 22
|
||||
assert item["create_time"] == 1740000000000
|
||||
|
||||
|
||||
async def test_groups_owner_fields_default_null(make_client):
|
||||
"""未回填/上游缺失的行走 schema 默认值 null(不报错、不缺 key)。"""
|
||||
|
||||
async def seed(session):
|
||||
session.add(ChatGroup(room_id="old@chatroom", channel="wechat",
|
||||
owner_account_id="wxid_y"))
|
||||
await session.commit()
|
||||
|
||||
client = await make_client(seed)
|
||||
r = await client.get("/api/groups", params={"room_id": "old@chatroom"})
|
||||
assert r.status_code == 200
|
||||
item = r.json()["items"][0]
|
||||
for key in ("owner_name", "owner_wxid", "owner_alias",
|
||||
"owner_avatar", "friend_count", "create_time"):
|
||||
assert item[key] is None
|
||||
|
||||
@@ -32,7 +32,7 @@ from typing import Any
|
||||
import httpx
|
||||
import respx
|
||||
|
||||
from app.scraper.wechat import scrape_wechat_groups
|
||||
from app.scraper.wechat import _normalize_chat_group, scrape_wechat_groups
|
||||
from app.session.yunke_client import YunkeClient
|
||||
from tests.cassettes.wxFriends_fixture import make_friend_list_response
|
||||
|
||||
@@ -277,3 +277,50 @@ async def test_scrape_wechat_groups_stops_on_empty_first_page():
|
||||
|
||||
assert results == []
|
||||
assert route.call_count == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 群主/建群补充列提取(2026-08-25 对齐云客员工聊天窗口·微信群 tab)
|
||||
# raw_json 键: ownerName/ownerWchatId/ownerAlias/ownerImage/friendCount/createTime
|
||||
# smoke.db 实测: createTime 全部为 'YYYY-MM-DD HH:mm:ss' 字符串;ms int 直通。
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_normalize_chat_group_extracts_owner_fields():
|
||||
"""6 个补充键提取到同名列;date-str createTime 归一为 ms epoch。"""
|
||||
record = {
|
||||
"friendWxId": "123@chatroom",
|
||||
"friendWxNickname": "测试群",
|
||||
"friendNum": 25,
|
||||
"wxId": "wxid_x",
|
||||
"ownerName": "Q**e",
|
||||
"ownerWchatId": "RL3*****108",
|
||||
"ownerAlias": "Fr******er",
|
||||
"ownerImage": "https://wx.qlogo.cn/o/0",
|
||||
"friendCount": 22,
|
||||
"createTime": "2025-10-27 13:30:55",
|
||||
}
|
||||
out = _normalize_chat_group(record)
|
||||
assert out["owner_name"] == "Q**e"
|
||||
assert out["owner_wxid"] == "RL3*****108"
|
||||
assert out["owner_alias"] == "Fr******er"
|
||||
assert out["owner_avatar"] == "https://wx.qlogo.cn/o/0"
|
||||
assert out["friend_count"] == 22
|
||||
# 'YYYY-MM-DD HH:mm:ss' -> ms epoch(naive UTC,同 addTime 约定)
|
||||
assert out["create_time"] == 1761571855000
|
||||
|
||||
|
||||
def test_normalize_chat_group_owner_fields_none_safe():
|
||||
"""缺失全部 6 个键 -> 全 None(老 capture 不炸)。"""
|
||||
out = _normalize_chat_group({"friendWxId": "r@chatroom"})
|
||||
for key in ("owner_name", "owner_wxid", "owner_alias",
|
||||
"owner_avatar", "friend_count", "create_time"):
|
||||
assert out[key] is None
|
||||
|
||||
|
||||
def test_normalize_chat_group_create_time_ms_passthrough():
|
||||
"""ms-epoch int 直通(不做日期解析)。"""
|
||||
out = _normalize_chat_group(
|
||||
{"friendWxId": "r@chatroom", "createTime": 1740000000000}
|
||||
)
|
||||
assert out["create_time"] == 1740000000000
|
||||
|
||||
Reference in New Issue
Block a user