yunke/yunke-server/app/scraper/parsers.py

"""Field / type-code normalization helpers for upstream record parsers.

The yunke CRM uses opaque integer type codes (``type`` / ``messageType`` on
WeChat, ``type`` / ``msgType`` on WhatsApp) on message records. The storage
layer prefers stable enum strings so that media dispatch (image / voice /
video / file) and presentation (text vs. date divider) can be keyed off a
single normalized value rather than the raw code.

This module is intentionally tiny: one total function per channel that maps
the Phase-0 confirmed codes and falls through to ``"unknown"`` for everything
else. Each MUST NOT raise — every record pulled during a scrape passes through
here, and one unexpected code must not abort the whole pull. The raw code is
preserved on the record's ``raw_json`` so the persistence layer can recover it
when needed.

:func:`coerce_int_or_none` is the shared scraper-layer int coercion (mirrors
:func:`app.storage.repos._coerce_int`) used to emit a ready-to-store
``type_raw`` int and to test raw type codes (e.g. the ``-2`` divider drop)
without the scraper taking a DB dependency.
"""

from __future__ import annotations

import datetime as dt
from collections.abc import Sequence
from typing import Any

__all__ = [
    "coerce_int_or_none",
    "normalize_wechat_type",
    "normalize_whatsapp_type",
    "parse_datetime_to_ms",
]

# Phase-0 confirmed WeChat message type codes (see
# docs/superpowers/specs/phase0-findings.md §2). Captured by live inspection
# of the yunke CRM.
# * ``1``  — plain text (and sticker text).
# * ``-2`` — date divider. NOT a real message; the scraper uses this label as
#   its drop signal (filtered out, never yielded, never persisted).
# * ``2``  — image / sticker (webp). ``file`` / ``fileTh`` are direct URLs;
#   download + dedupe by ``md5``.
# * ``3``  — image, legacy format. ``file`` is an opaque ID (not a URL) and
#   needs a separate download endpoint; flagged distinctly so the media layer
#   can route it.
# * ``4``  — video (mp4). ``file`` is a direct URL.
# * ``10`` — system / link / card / invite. ``file`` is a support URL (non-media);
#   the media layer skips downloading.
# Other codes (``6``, ``9``, ``13``, ``15``, ``21``, ``22`` — voice / file /
# transfer / redpacket / quote / etc.) are observed but unconfirmed; they pass
# through as ``"unknown"`` so the message is still ingested (raw code kept on
# ``raw_json``) rather than dropped. Extend this table as each is confirmed.
_WECHAT_TYPE_NAMES: dict[int, str] = {
    1: "text",
    -2: "divider",
    2: "image",
    3: "image_legacy",
    4: "video",
    10: "system",
}


def normalize_wechat_type(raw: Any) -> str:
    """Map an upstream WeChat ``type`` / ``messageType`` code to a stable string.

    Confirmed mappings: ``1→text``, ``-2→divider``, ``2→image``,
    ``3→image_legacy``, ``4→video``, ``10→system``. Any other / missing /
    unparsable value returns ``"unknown"``.

    Never raises: an unexpected type must not break the scrape. The raw code
    is preserved on ``raw_json`` so the persistence layer can recover it.
    """
    try:
        code = int(raw)
    except (TypeError, ValueError):
        return "unknown"
    return _WECHAT_TYPE_NAMES.get(code, "unknown")


# Phase-0 confirmed WhatsApp message type codes (see data dictionary §3 +
# phase0-findings.md §5). Captured by live inspection of the yunke CRM.
# * ``0``  — plain text.
# * ``1``  — image / media. ``mediaName`` is a deferred-download FILENAME
#   (not a URL); the media layer resolves it via a separate endpoint.
# * ``-2`` — date divider. NOT a real message; the scraper uses this label as
#   its drop signal (filtered out, never yielded, never persisted).
# Other codes pass through as ``"unknown"`` so the message is still ingested
# (raw code kept on ``raw_json``) rather than dropped. Extend this table as
# each is confirmed.
_WHATSAPP_TYPE_NAMES: dict[int, str] = {
    0: "text",
    1: "image",
    -2: "divider",
}


def normalize_whatsapp_type(raw: Any) -> str:
    """Map an upstream WhatsApp ``type`` / ``msgType`` code to a stable string.

    Confirmed mappings: ``0→text``, ``1→image``, ``-2→divider``. Any other /
    missing / unparsable value returns ``"unknown"``.

    Never raises: an unexpected type must not break the scrape. The raw code
    is preserved on ``raw_json`` so the persistence layer can recover it.
    """
    try:
        code = int(raw)
    except (TypeError, ValueError):
        return "unknown"
    return _WHATSAPP_TYPE_NAMES.get(code, "unknown")


def coerce_int_or_none(value: Any) -> int | None:
    """Coerce a possibly-string numeric value to ``int``; ``None`` preserved.

    Mirrors :func:`app.storage.repos._coerce_int` so the scraper can emit a
    ready-to-store ``type_raw`` int and check raw type codes (e.g. the ``-2``
    divider drop) without depending on the storage layer (the scraper has no
    DB dependency by design). A non-numeric value (``"text"`` / ``None`` /
    missing) returns ``None``.

    Kept in this module (rather than per-scraper) so both the WeChat and
    WhatsApp scrapers share one well-tested coercion instead of each carrying
    a private copy.
    """
    if value is None:
        return None
    if isinstance(value, bool):
        return int(value)
    if isinstance(value, int):
        return value
    if isinstance(value, float):
        return int(value)
    if isinstance(value, str):
        s = value.strip()
        if not s:
            return None
        try:
            return int(s)
        except ValueError:
            return None
    return None


def parse_datetime_to_ms(
    value: Any, formats: Sequence[str]
) -> int | None:
    """Coerce a datetime-shaped upstream value to a ms-epoch ``int``.

    Centralizes the datetime -> ms-epoch parsing shared by the call scrapers
    (``PhoneCall.start_call_time`` from ``callLog.startCallTime`` /
    ``Sms.create_time`` from ``smsLogsList.createTime``) and the voice-call
    scraper (``PhoneCall.start_call_time`` from ``startTime``). The accepted
    shapes:

    * ``int`` / ``float`` — treated as ms-epoch already, returned as ``int``.
    * numeric ``str`` (e.g. ``"1700000000123"``) -> ``int`` (no datetime
      round-trip).
    * datetime ``str`` matching one of ``formats`` -> parsed with
      ``datetime.strptime`` and converted to ms-epoch. The CRM is a Chinese
      deployment and the wall-clock times are presumably CST (UTC+8); this
      scraper parses them as **naive UTC** for determinism — a documented
      simplification (if production later needs CST-accurate buckets, attach
      a CST tzinfo to the parsed datetime before the timestamp call). The
      ``formats`` are tried in order; the first match wins. ``.replace(
      tzinfo=UTC)`` is chained on the strptime result so ruff's DTZ007 sees
      the naive datetime being made aware (and so the resulting epoch is
      well-defined).
    * ``None`` / empty / unparseable by any format -> ``None`` (no spurious
      value stored).

    ``formats`` is the ordered list of ``strptime`` patterns to attempt. The
    call scraper passes ``["%Y-%m-%d %H:%M:%S"]`` (``startCallTime``); the SMS
    scraper passes ``["%Y-%m-%d %H:%M"]`` (``createTime``, one precision step
    shorter — no seconds).
    """
    if value is None:
        return None
    if isinstance(value, bool):
        return int(value)
    if isinstance(value, (int, float)):
        return int(value)
    if isinstance(value, str):
        s = value.strip()
        if not s:
            return None
        # Numeric string -> ms-epoch int (no datetime round-trip).
        try:
            return int(s)
        except ValueError:
            pass
        # Try each strptime format in order; first match wins.
        for fmt in formats:
            try:
                parsed = dt.datetime.strptime(s, fmt).replace(tzinfo=dt.UTC)
            except ValueError:
                continue
            return int(parsed.timestamp() * 1000)
        return None
    return None