浅念
854efd3119
Adds the storage layer that bridges scrapers (which yield normalized dicts)
to the DB and the local media store.
app/storage/repos.py
* upsert_customer / upsert_customers / upsert_messages / upsert_wechat_account
/ upsert_staff_from_account, all taking an AsyncSession + scraper-shaped dicts.
* Dialect-aware Core upsert: SQLite on_conflict_do_update for tests,
MySQL on_duplicate_key_update for prod; dialect detected via session.bind.name.
_build_upsert_stmt(model, values, dialect) is the reusable helper.
* Type coercion: _coerce_bool / _coerce_int (module-level, unit-tested) map
is_send/is_del/is_block/is_on_job int-as-bool to bool and gender/type
numeric strings to int; None preserved; raw_json kept verbatim.
* For M1 the Message model maps to a single physical `message` table;
per-month sharding (message_table_name helper) is a LATER task (YAGNI).
app/storage/object_store.py
* Local-filesystem ObjectStore with put_bytes / put_stream / put_by_hash /
exists / get_path, all dispatched via asyncio.to_thread so disk I/O is
genuinely async.
* put_by_hash computes sha256 and writes only if absent (content-addressed
dedup); layout is base_dir/<first 2 hash chars>/<hash><ext>.
* Default base dir is Settings.media_dir; the interface is the seam a
future MinIO/S3 backend will swap in behind.
tests: 24 new tests (in-memory SQLite for repos; tmp_path for object_store)
covering idempotency (same PK twice -> 1 row, updated), type coercion, batch
dedup (3 messages / 1 dup msg_id -> 2 rows), and hash dedup (corrupt-file
probe proves second put_by_hash does not rewrite). ruff clean; full suite
67/67 green.
Co-Authored-By: Claude <noreply@anthropic.com>
157 lines
5.2 KiB
Python
157 lines
5.2 KiB
Python
"""Tests for app.storage.object_store — local-filesystem object store.
|
|
|
|
Exercises the async hash-dedup store against a real tmp_path on disk. All disk
|
|
I/O is dispatched via ``asyncio.to_thread`` so the methods are genuinely async.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from pathlib import Path
|
|
|
|
from app.storage.object_store import ObjectStore
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# put_bytes / get_path / exists
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
async def test_put_bytes_writes_file_and_exists_reports_true(tmp_path: Path):
|
|
"""put_bytes writes the blob; exists() then reports True; path matches key."""
|
|
store = ObjectStore(tmp_path)
|
|
key = "foo/bar.txt"
|
|
data = b"hello"
|
|
|
|
assert await store.exists(key) is False
|
|
|
|
path = await store.put_bytes(key, data)
|
|
assert path == tmp_path / key
|
|
assert path.exists()
|
|
assert path.read_bytes() == data
|
|
|
|
assert await store.exists(key) is True
|
|
|
|
|
|
async def test_get_path_returns_base_plus_key(tmp_path: Path):
|
|
"""get_path is a pure path computation (no I/O).
|
|
|
|
``/`` in the key becomes real path segments (subdirectories).
|
|
"""
|
|
store = ObjectStore(tmp_path)
|
|
assert store.get_path("a/b/c.bin") == tmp_path / "a" / "b" / "c.bin"
|
|
assert store.get_path("flat.txt") == tmp_path / "flat.txt"
|
|
|
|
|
|
async def test_put_bytes_overwrites_on_change(tmp_path: Path):
|
|
"""A second put_bytes with new content overwrites the file."""
|
|
store = ObjectStore(tmp_path)
|
|
await store.put_bytes("k", b"v1")
|
|
await store.put_bytes("k", b"v2")
|
|
assert (tmp_path / "k").read_bytes() == b"v2"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# put_by_hash (dedup)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
async def test_put_by_hash_returns_sha256_and_path_with_layout(tmp_path: Path):
|
|
"""Returns (sha256_hex, base/<2 chars>/<hash><ext>); file content correct."""
|
|
store = ObjectStore(tmp_path)
|
|
data = b"abc"
|
|
expected_hash = hashlib.sha256(data).hexdigest()
|
|
|
|
h, path = await store.put_by_hash(data, ext=".txt")
|
|
|
|
assert h == expected_hash
|
|
assert path == tmp_path / expected_hash[:2] / (expected_hash + ".txt")
|
|
assert path.exists()
|
|
assert path.read_bytes() == data
|
|
|
|
|
|
async def test_put_by_hash_dedups_same_bytes_returns_same_path(tmp_path: Path):
|
|
"""put_by_hash with identical bytes twice -> same (hash, path)."""
|
|
store = ObjectStore(tmp_path)
|
|
data = b"identical"
|
|
|
|
h1, p1 = await store.put_by_hash(data)
|
|
h2, p2 = await store.put_by_hash(data)
|
|
|
|
assert h1 == h2
|
|
assert p1 == p2
|
|
|
|
|
|
async def test_put_by_hash_does_not_rewrite_existing_file(tmp_path: Path):
|
|
"""Dedup: a second call for an existing hash must NOT touch the file.
|
|
|
|
Proves the implementation checks ``exists`` before writing. We corrupt the
|
|
file between calls; a dedup-aware store must leave the corruption in place
|
|
(no rewrite), since the blob is already present.
|
|
"""
|
|
store = ObjectStore(tmp_path)
|
|
data = b"original"
|
|
|
|
_h1, p1 = await store.put_by_hash(data, ext=".bin")
|
|
assert p1.read_bytes() == data
|
|
|
|
# Corrupt the on-disk file.
|
|
p1.write_bytes(b"CORRUPTED")
|
|
|
|
_h2, p2 = await store.put_by_hash(data, ext=".bin")
|
|
assert p2 == p1
|
|
# Dedup short-circuit: file content was NOT overwritten.
|
|
assert p2.read_bytes() == b"CORRUPTED"
|
|
|
|
|
|
async def test_put_by_hash_distinct_bytes_get_distinct_paths(tmp_path: Path):
|
|
"""Different bytes -> different hash -> different path."""
|
|
store = ObjectStore(tmp_path)
|
|
h1, p1 = await store.put_by_hash(b"aaa")
|
|
h2, p2 = await store.put_by_hash(b"bbb")
|
|
assert h1 != h2
|
|
assert p1 != p2
|
|
|
|
|
|
async def test_put_by_hash_default_ext_empty(tmp_path: Path):
|
|
"""Default ext='' -> no suffix appended to the file."""
|
|
store = ObjectStore(tmp_path)
|
|
data = b"x"
|
|
expected_hash = hashlib.sha256(data).hexdigest()
|
|
h, p = await store.put_by_hash(data)
|
|
assert h == expected_hash
|
|
assert p == tmp_path / expected_hash[:2] / expected_hash
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# put_stream
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
async def test_put_stream_writes_concatenated_chunks(tmp_path: Path):
|
|
"""put_stream consumes an async generator and writes concatenated bytes."""
|
|
|
|
async def gen():
|
|
yield b"chunk1-"
|
|
yield b"chunk2-"
|
|
yield b"chunk3"
|
|
|
|
store = ObjectStore(tmp_path)
|
|
path = await store.put_stream("streamed/key", gen())
|
|
assert path == tmp_path / "streamed" / "key"
|
|
assert path.read_bytes() == b"chunk1-chunk2-chunk3"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Default base dir (Settings.media_dir)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_object_store_default_base_uses_settings_media_dir(tmp_path: Path, monkeypatch):
|
|
"""Omitting base_dir falls back to Settings.media_dir."""
|
|
monkeypatch.setenv("MEDIA_DIR", str(tmp_path))
|
|
from app.config import get_settings
|
|
|
|
get_settings.cache_clear()
|
|
store = ObjectStore()
|
|
assert store.base_dir == get_settings().media_dir
|