From 8494d564ef84ff45a743e5312ba52d8b5ad06c7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B5=85=E5=BF=B5?= Date: Wed, 5 Aug 2026 18:44:04 +0800 Subject: [PATCH] feat(scraper): map customer add_time from raw_json Co-Authored-By: Claude --- yunke-server/app/scraper/wechat.py | 26 +++++++++++ yunke-server/app/scraper/whatsapp.py | 9 ++++ .../tests/test_scrape_wechat_friends.py | 46 +++++++++++++++++++ 3 files changed, 81 insertions(+) diff --git a/yunke-server/app/scraper/wechat.py b/yunke-server/app/scraper/wechat.py index c6386d5..6c6fe45 100644 --- a/yunke-server/app/scraper/wechat.py +++ b/yunke-server/app/scraper/wechat.py @@ -154,6 +154,28 @@ def _build_body( return body +# WeChat ``addTime`` is captured as ``'YYYY-MM-DD HH:mm:ss'`` (the common live +# shape; confirmed in smoke.db raw_json). Some captures instead carry a numeric +# ms-epoch. Both formats are accepted; first match wins. +_ADD_TIME_FORMATS = ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M") + + +def _parse_add_time(value: Any) -> int | None: + """Coerce a wechat ``addTime`` to a ms-epoch ``int`` (naive UTC). + + Accepted shapes: + + * ``int`` / ``float`` — treated as ms-epoch already, returned as ``int``. + * ``str`` matching one of :data:`_ADD_TIME_FORMATS` — parsed via the shared + :func:`app.scraper.parsers.parse_datetime_to_ms` (naive UTC, same + convention as ``lastUpdate`` / ``startTime`` elsewhere in this layer). + * numeric ``str`` (e.g. ``"1690000000000"``) -> ``int`` (no datetime + round-trip). + * ``None`` / empty / unparseable -> ``None`` (None-safe field). + """ + return parse_datetime_to_ms(value, _ADD_TIME_FORMATS) + + def _normalize(record: dict[str, Any]) -> dict[str, Any]: """Map one upstream friend record to the ``Customer`` column shape. @@ -181,6 +203,10 @@ def _normalize(record: dict[str, Any]) -> dict[str, Any]: # Prefer friendHeadImageUrl; fall back to the legacy headImg field. "head_image_url": record.get("friendHeadImageUrl") or record.get("headImg"), "last_chat_time": record.get("lastChatTime"), + # addTime is delivered as 'YYYY-MM-DD HH:mm:ss' (confirmed in smoke.db + # raw_json); some captures carry a numeric ms-epoch instead. Both shapes + # collapse to a ms-epoch int via _parse_add_time; None when missing/empty. + "add_time": _parse_add_time(record.get("addTime")), "send_num": record.get("sendNum"), "receive_num": record.get("receiveNum"), "message_count": record.get("messageCount"), diff --git a/yunke-server/app/scraper/whatsapp.py b/yunke-server/app/scraper/whatsapp.py index e29a971..8482277 100644 --- a/yunke-server/app/scraper/whatsapp.py +++ b/yunke-server/app/scraper/whatsapp.py @@ -248,6 +248,13 @@ def _normalize_whatsapp_friend(record: dict[str, Any]) -> dict[str, Any]: and the per-contact counts ``groupNum`` / ``contactsNum`` / ``talkerCount`` / ``friendAddNum`` / ``sensitiveWordNum`` survive end-to-end even though they have no direct ``Customer`` column). ``None``-safe on every field. + + ``add_time`` is deliberately ``None``: the WhatsApp ``businessFriendList`` + payload carries no customer-side add-time field (the ``createTime`` field + on this record is the **owner-staff** account's createTime, not the + contact's — confirmed via smoke.db raw_json dump). Mapping it to + ``createTime`` would be semantically wrong; the column stays NULL for WA + contacts (the wechat channel carries the truthful ``addTime``). """ return { "friend_id": record.get("friendUid"), @@ -259,6 +266,8 @@ def _normalize_whatsapp_friend(record: dict[str, Any]) -> dict[str, Any]: "user_id": record.get("userId"), "message_count": record.get("messageCount"), "last_chat_time": parse_datetime_to_ms(record.get("lastUpdate"), _WA_DATETIME_FORMATS), + # No customer-side add-time on this payload (see docstring); left NULL. + "add_time": None, "version": record.get("version"), "raw_json": record, } diff --git a/yunke-server/tests/test_scrape_wechat_friends.py b/yunke-server/tests/test_scrape_wechat_friends.py index d2eab65..a9c9751 100644 --- a/yunke-server/tests/test_scrape_wechat_friends.py +++ b/yunke-server/tests/test_scrape_wechat_friends.py @@ -152,3 +152,49 @@ async def test_scrape_wechat_friends_stops_on_empty_first_page(): assert results == [] assert route.call_count == 1 + + +# --------------------------------------------------------------------------- +# add_time mapping: addTime (string 'YYYY-MM-DD HH:mm:ss' OR ms-epoch int) +# --------------------------------------------------------------------------- + + +def test_normalize_maps_add_time_string_to_epoch(): + """addTime as a wall-clock string parses to a plausible recent ms epoch. + + The CRM delivers wechat ``addTime`` as ``'YYYY-MM-DD HH:mm:ss'`` (confirmed + in smoke.db raw_json). ``parse_datetime_to_ms`` parses naive UTC; we assert + only that the result is an int in the right magnitude for a 2026 date + (any 2026 moment > 1.76e12) rather than pinning a specific epoch so the + test is not brittle to the exact format-resolution / tz convention. + """ + from app.scraper.wechat import _normalize + + record = { + "friendWxId": "x", + "addTime": "2026-04-28 16:24:39", + "lastChatTime": 1700000000000, + } + out = _normalize(record) + assert isinstance(out["add_time"], int) + assert out["add_time"] > 1_760_000_000_000 + + +def test_normalize_maps_add_time_int_passthrough(): + """addTime as a numeric ms-epoch passes through verbatim (int or float).""" + from app.scraper.wechat import _normalize + + record = {"friendWxId": "x", "addTime": 1690000000000, "lastChatTime": None} + out = _normalize(record) + assert out["add_time"] == 1690000000000 + + +def test_normalize_add_time_none_when_missing(): + """Missing / empty addTime maps to None (None-safe field).""" + from app.scraper.wechat import _normalize + + out = _normalize({"friendWxId": "x"}) + assert out["add_time"] is None + + out_empty = _normalize({"friendWxId": "x", "addTime": ""}) + assert out_empty["add_time"] is None