"""Field / type-code normalization helpers for upstream record parsers. The yunke CRM uses opaque integer type codes (``type`` / ``messageType`` on WeChat, ``type`` / ``msgType`` on WhatsApp) on message records. The storage layer prefers stable enum strings so that media dispatch (image / voice / video / file) and presentation (text vs. date divider) can be keyed off a single normalized value rather than the raw code. This module is intentionally tiny: one total function per channel that maps the Phase-0 confirmed codes and falls through to ``"unknown"`` for everything else. Each MUST NOT raise — every record pulled during a scrape passes through here, and one unexpected code must not abort the whole pull. The raw code is preserved on the record's ``raw_json`` so the persistence layer can recover it when needed. :func:`coerce_int_or_none` is the shared scraper-layer int coercion (mirrors :func:`app.storage.repos._coerce_int`) used to emit a ready-to-store ``type_raw`` int and to test raw type codes (e.g. the ``-2`` divider drop) without the scraper taking a DB dependency. """ from __future__ import annotations import datetime as dt from collections.abc import Sequence from typing import Any __all__ = [ "coerce_int_or_none", "normalize_wechat_type", "normalize_whatsapp_type", "parse_datetime_to_ms", ] # Phase-0 confirmed WeChat message type codes (see # docs/superpowers/specs/phase0-findings.md §2). Captured by live inspection # of the yunke CRM. # * ``1`` — plain text (and sticker text). # * ``-2`` — date divider. NOT a real message; the scraper uses this label as # its drop signal (filtered out, never yielded, never persisted). # * ``2`` — image / sticker (webp). ``file`` / ``fileTh`` are direct URLs; # download + dedupe by ``md5``. # * ``3`` — image, legacy format. ``file`` is an opaque ID (not a URL) and # needs a separate download endpoint; flagged distinctly so the media layer # can route it. # * ``4`` — video (mp4). ``file`` is a direct URL. # * ``10`` — system / link / card / invite. ``file`` is a support URL (non-media); # the media layer skips downloading. # Other codes (``6``, ``9``, ``13``, ``15``, ``21``, ``22`` — voice / file / # transfer / redpacket / quote / etc.) are observed but unconfirmed; they pass # through as ``"unknown"`` so the message is still ingested (raw code kept on # ``raw_json``) rather than dropped. Extend this table as each is confirmed. _WECHAT_TYPE_NAMES: dict[int, str] = { 1: "text", -2: "divider", 2: "image", 3: "image_legacy", 4: "video", 10: "system", } def normalize_wechat_type(raw: Any) -> str: """Map an upstream WeChat ``type`` / ``messageType`` code to a stable string. Confirmed mappings: ``1→text``, ``-2→divider``, ``2→image``, ``3→image_legacy``, ``4→video``, ``10→system``. Any other / missing / unparsable value returns ``"unknown"``. Never raises: an unexpected type must not break the scrape. The raw code is preserved on ``raw_json`` so the persistence layer can recover it. """ try: code = int(raw) except (TypeError, ValueError): return "unknown" return _WECHAT_TYPE_NAMES.get(code, "unknown") # Phase-0 confirmed WhatsApp message type codes (see data dictionary §3 + # phase0-findings.md §5). Captured by live inspection of the yunke CRM. # * ``0`` — plain text. # * ``1`` — image / media. ``mediaName`` is a deferred-download FILENAME # (not a URL); the media layer resolves it via a separate endpoint. # * ``-2`` — date divider. NOT a real message; the scraper uses this label as # its drop signal (filtered out, never yielded, never persisted). # Other codes pass through as ``"unknown"`` so the message is still ingested # (raw code kept on ``raw_json``) rather than dropped. Extend this table as # each is confirmed. _WHATSAPP_TYPE_NAMES: dict[int, str] = { 0: "text", 1: "image", -2: "divider", } def normalize_whatsapp_type(raw: Any) -> str: """Map an upstream WhatsApp ``type`` / ``msgType`` code to a stable string. Confirmed mappings: ``0→text``, ``1→image``, ``-2→divider``. Any other / missing / unparsable value returns ``"unknown"``. Never raises: an unexpected type must not break the scrape. The raw code is preserved on ``raw_json`` so the persistence layer can recover it. """ try: code = int(raw) except (TypeError, ValueError): return "unknown" return _WHATSAPP_TYPE_NAMES.get(code, "unknown") def coerce_int_or_none(value: Any) -> int | None: """Coerce a possibly-string numeric value to ``int``; ``None`` preserved. Mirrors :func:`app.storage.repos._coerce_int` so the scraper can emit a ready-to-store ``type_raw`` int and check raw type codes (e.g. the ``-2`` divider drop) without depending on the storage layer (the scraper has no DB dependency by design). A non-numeric value (``"text"`` / ``None`` / missing) returns ``None``. Kept in this module (rather than per-scraper) so both the WeChat and WhatsApp scrapers share one well-tested coercion instead of each carrying a private copy. """ if value is None: return None if isinstance(value, bool): return int(value) if isinstance(value, int): return value if isinstance(value, float): return int(value) if isinstance(value, str): s = value.strip() if not s: return None try: return int(s) except ValueError: return None return None def parse_datetime_to_ms( value: Any, formats: Sequence[str] ) -> int | None: """Coerce a datetime-shaped upstream value to a ms-epoch ``int``. Centralizes the datetime -> ms-epoch parsing shared by the call scrapers (``PhoneCall.start_call_time`` from ``callLog.startCallTime`` / ``Sms.create_time`` from ``smsLogsList.createTime``) and the voice-call scraper (``PhoneCall.start_call_time`` from ``startTime``). The accepted shapes: * ``int`` / ``float`` — treated as ms-epoch already, returned as ``int``. * numeric ``str`` (e.g. ``"1700000000123"``) -> ``int`` (no datetime round-trip). * datetime ``str`` matching one of ``formats`` -> parsed with ``datetime.strptime`` and converted to ms-epoch. The CRM is a Chinese deployment and the wall-clock times are presumably CST (UTC+8); this scraper parses them as **naive UTC** for determinism — a documented simplification (if production later needs CST-accurate buckets, attach a CST tzinfo to the parsed datetime before the timestamp call). The ``formats`` are tried in order; the first match wins. ``.replace( tzinfo=UTC)`` is chained on the strptime result so ruff's DTZ007 sees the naive datetime being made aware (and so the resulting epoch is well-defined). * ``None`` / empty / unparseable by any format -> ``None`` (no spurious value stored). ``formats`` is the ordered list of ``strptime`` patterns to attempt. The call scraper passes ``["%Y-%m-%d %H:%M:%S"]`` (``startCallTime``); the SMS scraper passes ``["%Y-%m-%d %H:%M"]`` (``createTime``, one precision step shorter — no seconds). """ if value is None: return None if isinstance(value, bool): return int(value) if isinstance(value, (int, float)): return int(value) if isinstance(value, str): s = value.strip() if not s: return None # Numeric string -> ms-epoch int (no datetime round-trip). try: return int(s) except ValueError: pass # Try each strptime format in order; first match wins. for fmt in formats: try: parsed = dt.datetime.strptime(s, fmt).replace(tzinfo=dt.UTC) except ValueError: continue return int(parsed.timestamp() * 1000) return None return None