Initial commit: XHS crawler with dedup and content extraction

scripts/login_watcher.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+export DISPLAY=:99
+
+DATA_DIR="/app/data"
+REQ="${DATA_DIR}/.login_request.json"
+STATUS="${DATA_DIR}/.login_status.json"
+LOCK="${DATA_DIR}/.login_lock"
+
+write_status () {
+  local state="$1"
+  local msg="$2"
+  local ts
+  ts="$(date -Iseconds)"
+  python - <<PY > "$STATUS"
+import json
+print(json.dumps({"state":"$state","msg":"$msg","ts":"$ts"}, ensure_ascii=False))
+PY
+}
+
+echo "[login] watcher running, data_dir=${DATA_DIR}"
+
+while true; do
+  if [ -f "$REQ" ]; then
+    if [ -f "$LOCK" ]; then
+      sleep 1
+      continue
+    fi
+
+    touch "$LOCK"
+    write_status "running" "login started. Open noVNC and finish XHS login."
+
+    set +e
+    python -c "from xhs_crawler_core import login_mode_auto; login_mode_auto('data/storage_state.json', headless=False)"
+    rc=$?
+    set -e
+
+    if [ $rc -eq 0 ]; then
+      write_status "success" "login finished. storage_state.json updated."
+    else
+      write_status "failed" "login failed (rc=${rc}). Check xhs-novnc logs."
+    fi
+
+    rm -f "$REQ"
+    rm -f "$LOCK"
+  fi
+  sleep 1
+done

scripts/start_novnc.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -e
+
+export DISPLAY=:99
+mkdir -p /tmp/fluxbox
+
+# Start Xvfb
+Xvfb :99 -screen 0 1280x720x24 -nolisten tcp &
+
+# Wait for X socket
+for i in {1..50}; do
+  [ -S /tmp/.X11-unix/X99 ] && break
+  sleep 0.1
+done
+
+# Start window manager
+fluxbox -display :99 &
+
+# Start VNC server
+x11vnc -display :99 -nopw -forever -shared -rfbport 5900 &
+
+# Start noVNC web
+websockify --web=/usr/share/novnc/ 0.0.0.0:6080 localhost:5900 &
+
+# Start login watcher (MUST be before tail)
+if [ -x /app/login_watcher.sh ]; then
+  /app/login_watcher.sh &
+  echo "[login] watcher started"
+else
+  echo "[login] watcher missing: /app/login_watcher.sh"
+fi
+
+echo "[noVNC] Ready on :6080 (open /vnc_auto.html)"
+tail -f /dev/null

scripts/vnc_auto.html

@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html lang="zh-CN">
+<head>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width,initial-scale=1" />
+  <title>Auto VNC</title>
+  <style>
+    body{font-family:system-ui,-apple-system,Segoe UI,Roboto,Helvetica,Arial; padding:18px;}
+    .hint{color:#666;}
+    code{background:#f6f6f6; padding:2px 6px; border-radius:6px;}
+  </style>
+</head>
+<body>
+  <h3>正在进入登录桌面…</h3>
+  <p class="hint">如果没有自动跳转，请点击下面链接。</p>
+  <p><a id="go" href="vnc.html?autoconnect=1&resize=remote">进入 noVNC</a></p>
+  <script>
+    // Auto redirect in 300ms
+    setTimeout(function(){
+      location.href = document.getElementById('go').href;
+    }, 300);
+  </script>
+</body>
+</html>