806 lines
29 KiB
Python
806 lines
29 KiB
Python
|
||
import os
|
||
import json
|
||
import time
|
||
import threading
|
||
import queue
|
||
import pathlib
|
||
import zipfile
|
||
import re
|
||
from datetime import datetime
|
||
from typing import Any, Dict, List, Optional, Tuple
|
||
|
||
from openpyxl import load_workbook
|
||
|
||
from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException, UploadFile, File, Body
|
||
from fastapi.responses import HTMLResponse, FileResponse, JSONResponse
|
||
from fastapi.staticfiles import StaticFiles
|
||
|
||
# Import your existing crawler core
|
||
from xhs_crawler_core import (
|
||
crawl_detail,
|
||
check_login_state_with_storage,
|
||
login_mode_auto,
|
||
resolve_profile_by_search,
|
||
)
|
||
|
||
APP_ROOT = pathlib.Path(__file__).resolve().parent.parent
|
||
DATA_DIR = APP_ROOT / "data"
|
||
UPLOAD_DIR = DATA_DIR / "uploads"
|
||
OUT_DIR = APP_ROOT / "output" # 修改为 ./output/ 目录
|
||
LOG_DIR = APP_ROOT / "logs"
|
||
CRAWLED_IDS_PATH = DATA_DIR / "crawled_ids.json" # 已采集笔记ID记录
|
||
|
||
CONFIG_PATH = DATA_DIR / "config.json"
|
||
PROFILES_PATH = DATA_DIR / "profiles.json"
|
||
STORAGE_STATE_PATH_DEFAULT = DATA_DIR / "storage_state.json"
|
||
|
||
LOGIN_REQ_PATH = DATA_DIR / ".login_request.json"
|
||
LOGIN_STATUS_PATH = DATA_DIR / ".login_status.json"
|
||
LOGIN_LOCK_PATH = DATA_DIR / ".login_lock"
|
||
|
||
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
||
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||
UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
||
|
||
# ---- simple token auth (optional, lightweight) ----
|
||
ADMIN_TOKEN = os.getenv("ADMIN_TOKEN", "").strip()
|
||
|
||
def require_token(token: Optional[str]):
|
||
if ADMIN_TOKEN and token != ADMIN_TOKEN:
|
||
raise HTTPException(status_code=401, detail="Unauthorized")
|
||
|
||
# ---- logging hub (queue + websocket broadcast) ----
|
||
class LogHub:
|
||
def __init__(self):
|
||
self.q: "queue.Queue[str]" = queue.Queue()
|
||
self.clients: List[WebSocket] = []
|
||
self.lock = threading.Lock()
|
||
self._stop = threading.Event()
|
||
self._thread = threading.Thread(target=self._pump, daemon=True)
|
||
self._thread.start()
|
||
|
||
def emit(self, msg: str):
|
||
ts = time.strftime("%Y-%m-%d %H:%M:%S")
|
||
line = f"[{ts}] {msg}"
|
||
self.q.put(line)
|
||
# also append to file
|
||
try:
|
||
with open(LOG_DIR / "app.log", "a", encoding="utf-8") as f:
|
||
f.write(line + "\n")
|
||
except Exception:
|
||
pass
|
||
|
||
def attach(self, ws: WebSocket):
|
||
with self.lock:
|
||
self.clients.append(ws)
|
||
|
||
def detach(self, ws: WebSocket):
|
||
with self.lock:
|
||
if ws in self.clients:
|
||
self.clients.remove(ws)
|
||
|
||
def _pump(self):
|
||
while not self._stop.is_set():
|
||
try:
|
||
line = self.q.get(timeout=0.5)
|
||
except queue.Empty:
|
||
continue
|
||
# broadcast
|
||
dead = []
|
||
with self.lock:
|
||
clients = list(self.clients)
|
||
for ws in clients:
|
||
try:
|
||
# schedule send in event loop thread via ws.send_text (FastAPI handles)
|
||
# Here we just call it; if disconnected it will raise.
|
||
import anyio
|
||
anyio.from_thread.run(ws.send_text, line)
|
||
except Exception:
|
||
dead.append(ws)
|
||
for ws in dead:
|
||
self.detach(ws)
|
||
|
||
loghub = LogHub()
|
||
|
||
# ---- login state cache ----
|
||
LOGIN_CACHE_TTL = 60.0
|
||
_login_cache = {"ts": 0.0, "ok": False, "msg": "", "path": ""}
|
||
_login_lock = threading.Lock()
|
||
|
||
def _resolve_storage_path(cfg: Dict[str, Any]) -> str:
|
||
storage_path = cfg.get("storage_state_path") or str(STORAGE_STATE_PATH_DEFAULT)
|
||
if isinstance(storage_path, str) and storage_path.startswith("data/"):
|
||
storage_path = str(APP_ROOT / storage_path)
|
||
return storage_path
|
||
|
||
def _normalize_wait_range(value: Any, default_range: List[float]) -> tuple:
|
||
if isinstance(value, (list, tuple)) and len(value) == 2:
|
||
rng = value
|
||
else:
|
||
rng = default_range
|
||
try:
|
||
a = float(rng[0])
|
||
b = float(rng[1])
|
||
except Exception:
|
||
a = float(default_range[0])
|
||
b = float(default_range[1])
|
||
if a > b:
|
||
a, b = b, a
|
||
# Heuristic: treat small values as seconds and convert to ms.
|
||
if max(a, b) <= 60:
|
||
a *= 1000.0
|
||
b *= 1000.0
|
||
return (int(a), int(b))
|
||
|
||
def _check_login_cached(storage_path: str) -> Tuple[bool, str]:
|
||
now = time.time()
|
||
with _login_lock:
|
||
if _login_cache["path"] == storage_path and (now - _login_cache["ts"]) < LOGIN_CACHE_TTL:
|
||
return _login_cache["ok"], _login_cache["msg"]
|
||
ok, msg = check_login_state_with_storage(storage_path)
|
||
with _login_lock:
|
||
_login_cache.update({"ts": now, "ok": ok, "msg": msg, "path": storage_path})
|
||
return ok, msg
|
||
|
||
# ---- crawled IDs manager (去重管理) ----
|
||
class CrawledIDsManager:
|
||
"""管理已采集的笔记ID，用于去重"""
|
||
def __init__(self):
|
||
self.lock = threading.Lock()
|
||
self.ids: set = set()
|
||
self._load()
|
||
|
||
def _load(self):
|
||
"""从文件加载已采集ID"""
|
||
if CRAWLED_IDS_PATH.exists():
|
||
try:
|
||
with open(CRAWLED_IDS_PATH, "r", encoding="utf-8") as f:
|
||
data = json.load(f)
|
||
self.ids = set(data.get("ids", []))
|
||
loghub.emit(f"[去重] 已加载 {len(self.ids)} 条已采集笔记ID")
|
||
except Exception as e:
|
||
loghub.emit(f"[去重] 加载失败: {e}")
|
||
self.ids = set()
|
||
else:
|
||
self.ids = set()
|
||
self._save()
|
||
|
||
def _save(self):
|
||
"""保存已采集ID到文件"""
|
||
try:
|
||
CRAWLED_IDS_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||
with open(CRAWLED_IDS_PATH, "w", encoding="utf-8") as f:
|
||
json.dump({"ids": list(self.ids)}, f, ensure_ascii=False, indent=2)
|
||
except Exception as e:
|
||
loghub.emit(f"[去重] 保存失败: {e}")
|
||
|
||
def add(self, note_id: str):
|
||
"""添加已采集ID"""
|
||
with self.lock:
|
||
if note_id and note_id not in self.ids:
|
||
self.ids.add(note_id)
|
||
self._save()
|
||
|
||
def add_batch(self, note_ids: List[str]):
|
||
"""批量添加已采集ID"""
|
||
with self.lock:
|
||
for note_id in note_ids:
|
||
if note_id and note_id not in self.ids:
|
||
self.ids.add(note_id)
|
||
self._save()
|
||
|
||
def contains(self, note_id: str) -> bool:
|
||
"""检查是否已采集"""
|
||
with self.lock:
|
||
return note_id in self.ids
|
||
|
||
def clear(self):
|
||
"""清空所有记录"""
|
||
with self.lock:
|
||
count = len(self.ids)
|
||
self.ids.clear()
|
||
self._save()
|
||
loghub.emit(f"[去重] 已清空 {count} 条记录")
|
||
|
||
def count(self) -> int:
|
||
"""返回已采集ID数量"""
|
||
with self.lock:
|
||
return len(self.ids)
|
||
|
||
# 全局去重管理器
|
||
crawled_manager = CrawledIDsManager()
|
||
|
||
# ---- worker manager ----
|
||
class TaskManager:
|
||
def __init__(self):
|
||
self.running = False
|
||
self.stop_event = threading.Event()
|
||
self.thread: Optional[threading.Thread] = None
|
||
self.progress = {
|
||
"total": 0,
|
||
"done": 0,
|
||
"current": "",
|
||
"status": "idle", # idle/running/stopped/failed/done
|
||
"last_error": ""
|
||
}
|
||
|
||
def load_config(self) -> Dict[str, Any]:
|
||
if CONFIG_PATH.exists():
|
||
with open(CONFIG_PATH, "r", encoding="utf-8") as f:
|
||
return json.load(f)
|
||
return {}
|
||
|
||
def load_profiles(self) -> List[Dict[str, Any]]:
|
||
if PROFILES_PATH.exists():
|
||
with open(PROFILES_PATH, "r", encoding="utf-8") as f:
|
||
data = json.load(f)
|
||
if isinstance(data, list):
|
||
return data
|
||
return []
|
||
|
||
def _log(self, msg: str):
|
||
loghub.emit(msg)
|
||
|
||
def _run_one(self, profile: Dict[str, Any], cfg: Dict[str, Any]):
|
||
url = (profile.get("url") or "").strip()
|
||
xhs_no = (profile.get("xhs_no") or "").strip()
|
||
nickname = (profile.get("nickname") or "").strip()
|
||
who = xhs_no or nickname or url
|
||
if not url:
|
||
self._log(f"[跳过] 缺少 url: {profile}")
|
||
return
|
||
|
||
# 修改文件命名规则：账号ID_时间戳.xlsx
|
||
# 从 URL 中提取账号 ID
|
||
account_id = who
|
||
if "/user/profile/" in url:
|
||
account_id = url.split("/user/profile/")[-1].split("?")[0].strip()
|
||
elif who and who != url:
|
||
account_id = who
|
||
|
||
# 生成时间戳
|
||
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
|
||
out_name = f"{account_id}_{timestamp}.xlsx".replace("/", "_").replace("\\", "_")
|
||
out_path = str(OUT_DIR / out_name)
|
||
|
||
self._log(f"[开始] {who} -> {url}")
|
||
results = crawl_detail(
|
||
profile_url=url,
|
||
out_file=out_path,
|
||
limit=int(cfg.get("limit", 30)),
|
||
storage_state_path=_resolve_storage_path(cfg),
|
||
headless=bool(cfg.get("headless", True)),
|
||
slow_mo=int(cfg.get("slow_mo", 0)),
|
||
batch_min=int(cfg.get("batch_min", 12)),
|
||
batch_max=int(cfg.get("batch_max", 24)),
|
||
open_wait=_normalize_wait_range(cfg.get("open_wait"), [0.8, 2.0]),
|
||
between_wait=_normalize_wait_range(cfg.get("between_wait"), [0.4, 1.2]),
|
||
scroll_wait=_normalize_wait_range(cfg.get("scroll_wait"), [0.4, 1.1]),
|
||
long_break=_normalize_wait_range(cfg.get("long_break"), [8, 18]),
|
||
stop_event=self.stop_event,
|
||
log=lambda m: self._log(f"[{who}] {m}"),
|
||
dedup_check=crawled_manager.contains if cfg.get("enable_dedup", True) else None,
|
||
)
|
||
|
||
# 将采集成功的笔记ID添加到去重列表
|
||
if results and isinstance(results, list):
|
||
note_ids = []
|
||
for r in results:
|
||
href = r.get("href", "")
|
||
if href:
|
||
# 提取笔记ID（去除query参数）
|
||
note_id = href.split("?")[0].strip()
|
||
note_ids.append(note_id)
|
||
if note_ids:
|
||
crawled_manager.add_batch(note_ids)
|
||
self._log(f"[去重] 已添加 {len(note_ids)} 条笔记ID到去重库")
|
||
|
||
self._log(f"[完成] {who} 输出: {out_name}")
|
||
|
||
def _worker_loop(self, idx: int, q: "queue.Queue[Dict[str, Any]]", cfg: Dict[str, Any]):
|
||
while not self.stop_event.is_set():
|
||
try:
|
||
profile = q.get_nowait()
|
||
except queue.Empty:
|
||
return
|
||
try:
|
||
self.progress["current"] = profile.get("xhs_no") or profile.get("nickname") or profile.get("url") or ""
|
||
self._log(f"[W{idx}] 处理: {self.progress['current']}")
|
||
self._run_one(profile, cfg)
|
||
self.progress["done"] += 1
|
||
except Exception as e:
|
||
self.progress["last_error"] = str(e)
|
||
self._log(f"[W{idx}][错误] {e}")
|
||
finally:
|
||
q.task_done()
|
||
|
||
def start(self, profiles_override: Optional[List[Dict[str, Any]]] = None):
|
||
if self.running:
|
||
raise HTTPException(status_code=409, detail="Task already running")
|
||
cfg = self.load_config()
|
||
if profiles_override is not None:
|
||
if not isinstance(profiles_override, list):
|
||
raise HTTPException(status_code=400, detail="profiles must be a list")
|
||
profiles = list(profiles_override)
|
||
else:
|
||
profiles = self.load_profiles()
|
||
if not profiles:
|
||
raise HTTPException(status_code=400, detail="profiles.json 为空，请先导入账号列表")
|
||
# login check first
|
||
storage_path = _resolve_storage_path(cfg)
|
||
ok, msg = _check_login_cached(storage_path)
|
||
if not ok:
|
||
raise HTTPException(status_code=412, detail=msg or "登录态无效：请先在 noVNC 中完成登录并保存 storage_state.json")
|
||
|
||
self.stop_event.clear()
|
||
self.running = True
|
||
self.progress.update({"total": len(profiles), "done": 0, "current": "", "status": "running", "last_error": ""})
|
||
|
||
def run():
|
||
try:
|
||
self._log(f"[任务] 启动采集，总计 {len(profiles)} 个账号，并发={int(cfg.get('concurrency',2))}")
|
||
q: "queue.Queue[Dict[str, Any]]" = queue.Queue()
|
||
for p in profiles:
|
||
q.put(p)
|
||
|
||
concurrency = int(cfg.get("concurrency", 2))
|
||
workers = []
|
||
for i in range(concurrency):
|
||
t = threading.Thread(target=self._worker_loop, args=(i+1, q, cfg), daemon=True)
|
||
workers.append(t)
|
||
t.start()
|
||
|
||
for t in workers:
|
||
t.join()
|
||
|
||
if self.stop_event.is_set():
|
||
self.progress["status"] = "stopped"
|
||
self._log("[任务] 已停止")
|
||
else:
|
||
self.progress["status"] = "done"
|
||
self._log("[任务] 全部完成")
|
||
except Exception as e:
|
||
self.progress["status"] = "failed"
|
||
self.progress["last_error"] = str(e)
|
||
self._log(f"[任务][致命错误] {e}")
|
||
finally:
|
||
self.running = False
|
||
|
||
self.thread = threading.Thread(target=run, daemon=True)
|
||
self.thread.start()
|
||
|
||
def stop(self):
|
||
if not self.running:
|
||
return
|
||
self._log("[任务] 收到停止请求")
|
||
self.stop_event.set()
|
||
|
||
taskmgr = TaskManager()
|
||
|
||
def require_login_state() -> str:
|
||
cfg = taskmgr.load_config()
|
||
storage_path = _resolve_storage_path(cfg)
|
||
ok, msg = _check_login_cached(storage_path)
|
||
if not ok:
|
||
raise HTTPException(status_code=412, detail=msg or "登录状态无效，请先登录")
|
||
return storage_path
|
||
|
||
app = FastAPI(title="XHS Crawler Cloud")
|
||
|
||
app.mount("/static", StaticFiles(directory=str(APP_ROOT / "app" / "static")), name="static")
|
||
|
||
@app.get("/", response_class=HTMLResponse)
|
||
def index():
|
||
html_path = APP_ROOT / "app" / "static" / "index.html"
|
||
return HTMLResponse(html_path.read_text(encoding="utf-8"))
|
||
|
||
@app.get("/api/ping")
|
||
def ping():
|
||
return {"ok": True, "time": int(time.time())}
|
||
|
||
@app.get("/api/status")
|
||
def status(token: Optional[str] = None):
|
||
require_token(token)
|
||
cfg = taskmgr.load_config()
|
||
storage_path = _resolve_storage_path(cfg)
|
||
login_ok = False
|
||
login_msg = ""
|
||
try:
|
||
login_ok, login_msg = _check_login_cached(storage_path)
|
||
except Exception as e:
|
||
loghub.emit(f"[状态] 检查登录态异常: {e}")
|
||
login_msg = str(e)
|
||
|
||
return {
|
||
"running": taskmgr.running,
|
||
"progress": taskmgr.progress,
|
||
"login_ok": login_ok,
|
||
"login_msg": login_msg,
|
||
"storage_state_path": storage_path,
|
||
}
|
||
|
||
@app.get("/api/config")
|
||
def get_config(token: Optional[str] = None):
|
||
require_token(token)
|
||
require_login_state()
|
||
return taskmgr.load_config()
|
||
|
||
@app.post("/api/config")
|
||
def set_config(payload: Dict[str, Any], token: Optional[str] = None):
|
||
require_token(token)
|
||
require_login_state()
|
||
with open(CONFIG_PATH, "w", encoding="utf-8") as f:
|
||
json.dump(payload, f, ensure_ascii=False, indent=2)
|
||
loghub.emit("[配置] 已保存 config.json")
|
||
return {"ok": True}
|
||
|
||
@app.get("/api/profiles")
|
||
def get_profiles(token: Optional[str] = None):
|
||
require_token(token)
|
||
require_login_state()
|
||
return taskmgr.load_profiles()
|
||
|
||
@app.post("/api/profiles")
|
||
def set_profiles(payload: List[Dict[str, Any]], token: Optional[str] = None):
|
||
require_token(token)
|
||
require_login_state()
|
||
with open(PROFILES_PATH, "w", encoding="utf-8") as f:
|
||
json.dump(payload, f, ensure_ascii=False, indent=2)
|
||
loghub.emit(f"[Profiles] 已保存 profiles.json，共 {len(payload)} 条")
|
||
return {"ok": True}
|
||
|
||
def _merge_profile(existing: List[Dict[str, Any]], profile: Dict[str, Any]) -> Tuple[List[Dict[str, Any]], int, int]:
|
||
url = (profile.get("url") or "").strip()
|
||
if not url:
|
||
return existing, 0, 0
|
||
for p in existing:
|
||
if (p.get("url") or "").strip() == url:
|
||
updated = False
|
||
for k in ("xhs_no", "nickname", "url"):
|
||
v = (profile.get(k) or "").strip()
|
||
if v and v != (p.get(k) or ""):
|
||
p[k] = v
|
||
updated = True
|
||
return existing, 0, 1 if updated else 0
|
||
existing.append(profile)
|
||
return existing, 1, 0
|
||
|
||
@app.post("/api/profiles/resolve")
|
||
def resolve_profile(payload: Dict[str, Any], token: Optional[str] = None):
|
||
require_token(token)
|
||
storage_path = require_login_state()
|
||
keyword = str(payload.get("keyword", "")).strip()
|
||
if not keyword:
|
||
raise HTTPException(status_code=400, detail="keyword required")
|
||
headless = bool(payload.get("headless", True))
|
||
slow_mo = int(payload.get("slow_mo", 0))
|
||
try:
|
||
profile = resolve_profile_by_search(
|
||
keyword=keyword,
|
||
storage_state_path=storage_path,
|
||
headless=headless,
|
||
slow_mo=slow_mo,
|
||
log=loghub.emit,
|
||
)
|
||
except Exception as e:
|
||
loghub.emit(f"[解析][错误] {e}")
|
||
raise HTTPException(status_code=400, detail=str(e))
|
||
existing = taskmgr.load_profiles()
|
||
merged, added, updated = _merge_profile(existing, profile)
|
||
with open(PROFILES_PATH, "w", encoding="utf-8") as f:
|
||
json.dump(merged, f, ensure_ascii=False, indent=2)
|
||
if added:
|
||
loghub.emit(f"[Profiles] 解析新增账号：{profile.get('xhs_no') or profile.get('url')}")
|
||
elif updated:
|
||
loghub.emit(f"[Profiles] 更新账号信息：{profile.get('xhs_no') or profile.get('url')}")
|
||
return {"ok": True, "profile": profile, "added": added, "updated": updated, "profiles": merged}
|
||
# ---- Excel import helpers (profiles) ----
|
||
def _excel_headers(path: pathlib.Path, sheet: str) -> List[str]:
|
||
wb = load_workbook(filename=str(path), read_only=True, data_only=True)
|
||
if sheet not in wb.sheetnames:
|
||
raise HTTPException(status_code=400, detail=f"sheet not found: {sheet}")
|
||
ws = wb[sheet]
|
||
# find first non-empty row as header
|
||
header = []
|
||
for row in ws.iter_rows(min_row=1, max_row=20, values_only=True):
|
||
if row and any(v is not None and str(v).strip() for v in row):
|
||
header = [str(v).strip() if v is not None else "" for v in row]
|
||
break
|
||
wb.close()
|
||
# sanitize
|
||
header = [h for h in header if h]
|
||
return header
|
||
|
||
def _excel_column_values(path: pathlib.Path, sheet: str, column_key: str) -> List[str]:
|
||
wb = load_workbook(filename=str(path), read_only=True, data_only=True)
|
||
if sheet not in wb.sheetnames:
|
||
raise HTTPException(status_code=400, detail=f"sheet not found: {sheet}")
|
||
ws = wb[sheet]
|
||
# header row
|
||
header_row = None
|
||
header_vals = None
|
||
for ridx, row in enumerate(ws.iter_rows(min_row=1, max_row=50, values_only=True), start=1):
|
||
if row and any(v is not None and str(v).strip() for v in row):
|
||
header_row = ridx
|
||
header_vals = [str(v).strip() if v is not None else "" for v in row]
|
||
break
|
||
if not header_row or not header_vals:
|
||
wb.close()
|
||
return []
|
||
# locate column index by header text OR excel col letter
|
||
col_idx = None
|
||
if re.fullmatch(r"[A-Za-z]{1,3}", column_key.strip() or ""):
|
||
# letter
|
||
letters = column_key.strip().upper()
|
||
col_idx = 0
|
||
for ch in letters:
|
||
col_idx = col_idx * 26 + (ord(ch) - ord('A') + 1)
|
||
else:
|
||
for i, h in enumerate(header_vals, start=1):
|
||
if h == column_key:
|
||
col_idx = i
|
||
break
|
||
if not col_idx:
|
||
wb.close()
|
||
raise HTTPException(status_code=400, detail=f"column not found: {column_key}")
|
||
values: List[str] = []
|
||
for row in ws.iter_rows(min_row=header_row+1, values_only=True):
|
||
if not row:
|
||
continue
|
||
v = row[col_idx-1] if col_idx-1 < len(row) else None
|
||
if v is None:
|
||
continue
|
||
s = str(v).strip()
|
||
if not s:
|
||
continue
|
||
values.append(s)
|
||
wb.close()
|
||
# unique keep order
|
||
seen = set()
|
||
out = []
|
||
for s in values:
|
||
if s in seen:
|
||
continue
|
||
seen.add(s)
|
||
out.append(s)
|
||
return out
|
||
|
||
# Keep uploaded excel file records in memory (simple)
|
||
_excel_files: Dict[str, str] = {}
|
||
_excel_lock = threading.Lock()
|
||
|
||
@app.post("/api/excel/upload")
|
||
def excel_upload(file: UploadFile = File(...), token: Optional[str] = None):
|
||
require_token(token)
|
||
require_login_state()
|
||
if not file.filename.lower().endswith((".xlsx", ".xlsm", ".xltx", ".xltm")):
|
||
raise HTTPException(status_code=400, detail="Only .xlsx/.xlsm supported")
|
||
ts = int(time.time())
|
||
fid = f"{ts}_{abs(hash(file.filename))%1000000}"
|
||
dst = UPLOAD_DIR / f"{fid}.xlsx"
|
||
content = file.file.read()
|
||
dst.write_bytes(content)
|
||
wb = load_workbook(filename=str(dst), read_only=True, data_only=True)
|
||
sheets = list(wb.sheetnames)
|
||
wb.close()
|
||
with _excel_lock:
|
||
_excel_files[fid] = str(dst)
|
||
# return columns for first sheet as convenience
|
||
cols = _excel_headers(dst, sheets[0]) if sheets else []
|
||
loghub.emit(f"[Excel] 上传 {file.filename} -> {dst.name}")
|
||
return {"file_id": fid, "sheets": sheets, "columns": cols, "default_sheet": sheets[0] if sheets else ""}
|
||
|
||
@app.get("/api/excel/meta")
|
||
def excel_meta(file_id: str, sheet: str, token: Optional[str] = None):
|
||
require_token(token)
|
||
require_login_state()
|
||
with _excel_lock:
|
||
path_str = _excel_files.get(file_id)
|
||
if not path_str or not os.path.exists(path_str):
|
||
raise HTTPException(status_code=404, detail="excel file_id not found")
|
||
cols = _excel_headers(pathlib.Path(path_str), sheet)
|
||
return {"file_id": file_id, "sheet": sheet, "columns": cols}
|
||
|
||
@app.get("/api/excel/column_values")
|
||
def excel_column_values(file_id: str, sheet: str, column: str, token: Optional[str] = None):
|
||
require_token(token)
|
||
require_login_state()
|
||
with _excel_lock:
|
||
path_str = _excel_files.get(file_id)
|
||
if not path_str or not os.path.exists(path_str):
|
||
raise HTTPException(status_code=404, detail="excel file_id not found")
|
||
vals = _excel_column_values(pathlib.Path(path_str), sheet, column)
|
||
loghub.emit(f"[Excel] 读取列 {column} -> {len(vals)} 条")
|
||
return {"file_id": file_id, "sheet": sheet, "column": column, "values": vals}
|
||
|
||
def _value_to_profile(v: str) -> Dict[str, Any]:
|
||
s = (v or "").strip()
|
||
if not s:
|
||
return {}
|
||
if s.startswith("http://") or s.startswith("https://"):
|
||
return {"xhs_no": "", "nickname": "", "url": s}
|
||
# treat as XHS user id / short id
|
||
url = f"https://www.xiaohongshu.com/user/profile/{s}"
|
||
return {"xhs_no": s, "nickname": "", "url": url}
|
||
|
||
@app.post("/api/profiles/import_excel")
|
||
def profiles_import_excel(payload: Dict[str, Any], token: Optional[str] = None):
|
||
"""
|
||
payload: {file_id, sheet, column}
|
||
column can be header text or Excel column letter (A/B/C...)
|
||
"""
|
||
require_token(token)
|
||
require_login_state()
|
||
file_id = str(payload.get("file_id","")).strip()
|
||
sheet = str(payload.get("sheet","")).strip()
|
||
column = str(payload.get("column","")).strip()
|
||
if not (file_id and sheet and column):
|
||
raise HTTPException(status_code=400, detail="file_id/sheet/column required")
|
||
with _excel_lock:
|
||
path_str = _excel_files.get(file_id)
|
||
if not path_str or not os.path.exists(path_str):
|
||
raise HTTPException(status_code=404, detail="excel file_id not found")
|
||
vals = _excel_column_values(pathlib.Path(path_str), sheet, column)
|
||
new_profiles = []
|
||
for v in vals:
|
||
p = _value_to_profile(v)
|
||
if p:
|
||
new_profiles.append(p)
|
||
# merge by url
|
||
existing = taskmgr.load_profiles()
|
||
seen = set((p.get("url") or "").strip() for p in existing if (p.get("url") or "").strip())
|
||
merged = list(existing)
|
||
add_count = 0
|
||
for p in new_profiles:
|
||
u = (p.get("url") or "").strip()
|
||
if not u or u in seen:
|
||
continue
|
||
merged.append(p)
|
||
seen.add(u)
|
||
add_count += 1
|
||
with open(PROFILES_PATH, "w", encoding="utf-8") as f:
|
||
json.dump(merged, f, ensure_ascii=False, indent=2)
|
||
loghub.emit(f"[Profiles] Excel 导入完成：新增 {add_count} 条（总计 {len(merged)}）")
|
||
return {"ok": True, "added": add_count, "total": len(merged), "profiles": merged}
|
||
|
||
# ---- multiple storage_state accounts (optional) ----
|
||
@app.get("/api/accounts")
|
||
def list_accounts(token: Optional[str] = None):
|
||
require_token(token)
|
||
require_login_state()
|
||
items = []
|
||
for p in sorted(DATA_DIR.glob("storage_state*.json")):
|
||
items.append({"name": p.name, "path": str(p), "mtime": int(p.stat().st_mtime)})
|
||
return items
|
||
|
||
@app.post("/api/accounts/select")
|
||
def select_account(payload: Dict[str, Any], token: Optional[str] = None):
|
||
require_token(token)
|
||
require_login_state()
|
||
name = str(payload.get("name","")).strip()
|
||
if not name:
|
||
raise HTTPException(status_code=400, detail="name required")
|
||
p = DATA_DIR / name
|
||
if not p.exists():
|
||
raise HTTPException(status_code=404, detail="storage_state file not found")
|
||
cfg = taskmgr.load_config()
|
||
cfg["storage_state_path"] = f"data/{name}"
|
||
with open(CONFIG_PATH, "w", encoding="utf-8") as f:
|
||
json.dump(cfg, f, ensure_ascii=False, indent=2)
|
||
loghub.emit(f"[登录] 已切换采集账号：{name}")
|
||
return {"ok": True, "storage_state_path": cfg["storage_state_path"]}
|
||
|
||
|
||
@app.post("/api/start")
|
||
def start(payload: Optional[Dict[str, Any]] = Body(default=None), token: Optional[str] = None):
|
||
require_token(token)
|
||
require_login_state()
|
||
profiles_override = None
|
||
if isinstance(payload, dict) and payload.get("profiles") is not None:
|
||
if not isinstance(payload.get("profiles"), list):
|
||
raise HTTPException(status_code=400, detail="profiles must be a list")
|
||
profiles_override = payload.get("profiles")
|
||
taskmgr.start(profiles_override=profiles_override)
|
||
return {"ok": True}
|
||
|
||
@app.post("/api/stop")
|
||
def stop(token: Optional[str] = None):
|
||
require_token(token)
|
||
require_login_state()
|
||
taskmgr.stop()
|
||
return {"ok": True}
|
||
|
||
@app.get("/api/files")
|
||
def list_files(token: Optional[str] = None):
|
||
require_token(token)
|
||
require_login_state()
|
||
items = []
|
||
for p in sorted(OUT_DIR.glob("*")):
|
||
if p.is_file():
|
||
items.append({"name": p.name, "size": p.stat().st_size, "mtime": int(p.stat().st_mtime)})
|
||
return items
|
||
|
||
@app.get("/api/files/{name}")
|
||
def download_file(name: str, token: Optional[str] = None):
|
||
require_token(token)
|
||
require_login_state()
|
||
p = OUT_DIR / name
|
||
if not p.exists() or not p.is_file():
|
||
raise HTTPException(status_code=404, detail="file not found")
|
||
return FileResponse(str(p), filename=p.name)
|
||
|
||
@app.get("/api/files_zip")
|
||
def download_zip(token: Optional[str] = None):
|
||
require_token(token)
|
||
require_login_state()
|
||
zip_path = OUT_DIR / "out_bundle.zip"
|
||
with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as z:
|
||
for p in OUT_DIR.glob("*.xlsx"):
|
||
z.write(p, arcname=p.name)
|
||
return FileResponse(str(zip_path), filename="out_bundle.zip")
|
||
|
||
@app.post("/api/dedup/clear")
|
||
def clear_dedup(token: Optional[str] = None):
|
||
"""清空去重记录"""
|
||
require_token(token)
|
||
require_login_state()
|
||
count = crawled_manager.count()
|
||
crawled_manager.clear()
|
||
loghub.emit(f"[去重] 已通过接口清空 {count} 条记录")
|
||
return {"ok": True, "cleared": count}
|
||
|
||
@app.get("/api/dedup/status")
|
||
def dedup_status(token: Optional[str] = None):
|
||
"""查询去重状态"""
|
||
require_token(token)
|
||
require_login_state()
|
||
return {"ok": True, "count": crawled_manager.count()}
|
||
|
||
|
||
@app.post("/api/login/start")
|
||
def login_start(token: Optional[str] = None):
|
||
require_token(token)
|
||
if LOGIN_LOCK_PATH.exists():
|
||
raise HTTPException(status_code=409, detail="login already running, please open noVNC to finish it")
|
||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||
UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
||
LOGIN_REQ_PATH.write_text(json.dumps({"ts": int(time.time())}, ensure_ascii=False), encoding="utf-8")
|
||
return {"ok": True, "msg": "login requested. Open noVNC and scan QR code."}
|
||
|
||
@app.get("/api/login/status")
|
||
def login_status(token: Optional[str] = None):
|
||
require_token(token)
|
||
if LOGIN_STATUS_PATH.exists():
|
||
try:
|
||
return json.loads(LOGIN_STATUS_PATH.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
return {"state": "unknown", "msg": "status parse error"}
|
||
if LOGIN_LOCK_PATH.exists():
|
||
return {"state": "running", "msg": "login running, open noVNC"}
|
||
return {"state": "idle", "msg": "no login yet"}
|
||
|
||
|
||
@app.websocket("/ws/logs")
|
||
async def ws_logs(ws: WebSocket):
|
||
await ws.accept()
|
||
loghub.attach(ws)
|
||
loghub.emit("[WS] 日志连接已建立")
|
||
try:
|
||
while True:
|
||
# keepalive / receive (client may send ping)
|
||
await ws.receive_text()
|
||
except WebSocketDisconnect:
|
||
loghub.emit("[WS] 日志连接已断开")
|
||
finally:
|
||
loghub.detach(ws)
|
||
|
||
@app.get("/api/login_hint")
|
||
def login_hint(token: Optional[str] = None):
|
||
require_token(token)
|
||
return {
|
||
"novnc_url": os.getenv("NOVNC_URL", "http://219.157.255.213:26080/vnc_auto.html"),
|
||
"note": "如果登录态无效，请打开 noVNC 登录窗口，完成扫码/验证后再回来点击“重新检查登录状态”。"
|
||
}
|