yunke/yunke-server/app/scraper/parsers.py

"""Field / type-code normalization helpers for upstream record parsers.

The yunke CRM uses opaque integer type codes (``type`` / ``messageType``) on
WeChat message records. The storage layer prefers stable enum strings so that
media dispatch (image / voice / video / file) and presentation (text vs.
date divider) can be keyed off a single normalized value rather than the
raw code.

This module is intentionally tiny: one total function that maps the
Phase-0 confirmed codes and falls through to ``"unknown"`` for everything
else. It MUST NOT raise — every record pulled during a scrape passes through
here, and one unexpected code must not abort the whole pull. The raw code is
preserved on the record's ``raw_json`` so the persistence layer can recover
it when needed.
"""

from __future__ import annotations

from typing import Any

__all__ = ["normalize_wechat_type"]

# Phase-0 confirmed WeChat message type codes (see
# docs/superpowers/specs/phase0-findings.md §2). Captured by live inspection
# of the yunke CRM.
# * ``1``  — plain text (and sticker text).
# * ``-2`` — date divider. NOT a real message; the scraper uses this label as
#   its drop signal (filtered out, never yielded, never persisted).
# * ``2``  — image / sticker (webp). ``file`` / ``fileTh`` are direct URLs;
#   download + dedupe by ``md5``.
# * ``3``  — image, legacy format. ``file`` is an opaque ID (not a URL) and
#   needs a separate download endpoint; flagged distinctly so the media layer
#   can route it.
# * ``4``  — video (mp4). ``file`` is a direct URL.
# * ``10`` — system / link / card / invite. ``file`` is a support URL (non-media);
#   the media layer skips downloading.
# Other codes (``6``, ``9``, ``13``, ``15``, ``21``, ``22`` — voice / file /
# transfer / redpacket / quote / etc.) are observed but unconfirmed; they pass
# through as ``"unknown"`` so the message is still ingested (raw code kept on
# ``raw_json``) rather than dropped. Extend this table as each is confirmed.
_WECHAT_TYPE_NAMES: dict[int, str] = {
    1: "text",
    -2: "divider",
    2: "image",
    3: "image_legacy",
    4: "video",
    10: "system",
}


def normalize_wechat_type(raw: Any) -> str:
    """Map an upstream WeChat ``type`` / ``messageType`` code to a stable string.

    Confirmed mappings: ``1→text``, ``-2→divider``, ``2→image``,
    ``3→image_legacy``, ``4→video``, ``10→system``. Any other / missing /
    unparsable value returns ``"unknown"``.

    Never raises: an unexpected type must not break the scrape. The raw code
    is preserved on ``raw_json`` so the persistence layer can recover it.
    """
    try:
        code = int(raw)
    except (TypeError, ValueError):
        return "unknown"
    return _WECHAT_TYPE_NAMES.get(code, "unknown")