hermes
647 lines
36 KiB
JavaScript
Executable File
647 lines
36 KiB
JavaScript
Executable File
#!/usr/bin/env node
|
||
// zhaopin-scrape.js v10 — 智联招聘推荐页候选人抓取 + 硬性预筛 + 七维评分 + 弹窗内打招呼
|
||
//
|
||
// v10 变更（2026-08-26）：
|
||
// 1. 集成 scoring.js 硬性预筛门槛：无外贸岗位/语言专业/指定证书、毕业年份不在
|
||
// 2020-2026、年龄>35 的候选人 → 标"不相关/不推荐"，绝不打招呼（解决"打招呼范围过宽"）
|
||
// 2. 依赖去硬编码：playwright-core / better-sqlite3 依次尝试
|
||
// 环境变量 PWCORE_PATH/SQLITE3_PATH → 脚本目录 node_modules → 全局 require
|
||
// 3. 数据库路径默认 ./recruitment.db（RECRUITMENT_DB 覆盖），不再写死 /home/kt
|
||
// 4. 保留 v9.2 已验证行为：resumeNumber 去重、nextElementSibling 导航、
|
||
// 弹窗内打招呼、聊天次数监控、公司黑名单、人类行为模拟、健康检测熔断
|
||
//
|
||
// 运行：ACCOUNT=hr1 CDP_PORT=9223 node zhaopin-scrape.js [数量] [--full]
|
||
// --full 全量模式（重抓已存储者）；默认增量
|
||
// --score-only 只抓取评分，不打招呼（验证新门槛时用）
|
||
|
||
'use strict';
|
||
|
||
// ---------- 依赖加载（多重回退，勿写死绝对路径） ----------
|
||
|
||
function tryRequire(names) {
|
||
for (const n of names) {
|
||
try { return require(n); } catch (_) { /* 下一个 */ }
|
||
}
|
||
throw new Error(`无法加载依赖，请 npm install playwright-core better-sqlite3，或设置 PWCORE_PATH/SQLITE3_PATH 环境变量`);
|
||
}
|
||
|
||
const path = require('path');
|
||
const crypto = require('crypto');
|
||
const { chromium } = tryRequire([
|
||
process.env.PWCORE_PATH,
|
||
path.join(__dirname, 'node_modules', 'playwright-core'),
|
||
'playwright-core',
|
||
].filter(Boolean));
|
||
const Database = tryRequire([
|
||
process.env.SQLITE3_PATH,
|
||
path.join(__dirname, 'node_modules', 'better-sqlite3'),
|
||
'better-sqlite3',
|
||
].filter(Boolean));
|
||
|
||
const { evaluateCandidate, parseExperienceYears, CONFIG } = require('./scoring');
|
||
|
||
// ---------- 环境配置 ----------
|
||
|
||
const DB_PATH = process.env.RECRUITMENT_DB || path.join(__dirname, 'recruitment.db');
|
||
const CDP_HOST = process.env.CDP_HOST || 'localhost';
|
||
const CDP_PORT = process.env.CDP_PORT || '9222';
|
||
const ACCOUNT = process.env.ACCOUNT || 'hr1';
|
||
const TARGET_CITY = process.env.TARGET_CITY || '郑州';
|
||
const COMPANY_BLACKLIST = ['合塑', '派卡']; // 从 workDetail 提取公司名命中即跳过
|
||
|
||
// ---------- 个人黑名单（v10.13, 2026-09-29 二次修订：多维度判定）----------
|
||
// person_blacklist 表：用户指定的候选人。
|
||
// 判定原则（用户定稿）：必须多维度交叉命中才认定同人，单一维度不触发，防止误伤。
|
||
// 维度：① resumeNumber（平台唯一ID，单独即可判定）② 全名 ③ 学校 ④ 工作单位 ⑤ 其他特征词
|
||
// 规则：resumeNumber 命中 → 同人；否则 全名+任一特征词 或 学校+工作单位 等任意 ≥2 维命中 → 同人
|
||
let PERSON_BLACKLIST = [];
|
||
function loadPersonBlacklist(db) {
|
||
try {
|
||
PERSON_BLACKLIST = db.prepare('SELECT * FROM person_blacklist').all().map(p => ({
|
||
...p,
|
||
fields: JSON.parse(p.match_fields || '[]'),
|
||
}));
|
||
if (PERSON_BLACKLIST.length) log(`个人黑名单已加载: ${PERSON_BLACKLIST.length} 人 [${PERSON_BLACKLIST.map(p => p.name).join('、')}]`);
|
||
} catch (e) { PERSON_BLACKLIST = []; }
|
||
}
|
||
// 弹窗级多维度匹配：统计命中维数，≥2 才认定
|
||
function personBlackHitModal(fullName, resumeNumber, rawText) {
|
||
for (const p of PERSON_BLACKLIST) {
|
||
if (p.resume_number && resumeNumber && p.resume_number === resumeNumber) return `${p.name}(唯一ID)`;
|
||
const dims = [];
|
||
if (fullName && p.name && fullName === p.name) dims.push('全名');
|
||
if (p.fields.length && rawText) {
|
||
for (const f of p.fields) if (rawText.includes(f)) dims.push(f.length > 8 ? f.slice(0, 8) + '…' : f);
|
||
}
|
||
if (dims.length >= 2) return `${p.name}(多维命中:${dims.join('+')})`;
|
||
}
|
||
return null;
|
||
}
|
||
const MAX_START_SKIP = 50;
|
||
|
||
// ---------- 工具 ----------
|
||
|
||
function randomSleep(minSec, maxSec) {
|
||
const ms = Math.floor(Math.random() * (maxSec - minSec + 1) + minSec) * 1000;
|
||
return new Promise(r => setTimeout(r, ms));
|
||
}
|
||
|
||
function fingerprint(name, age, education, experience, school) {
|
||
return crypto.createHash('md5')
|
||
.update([name, age, education, experience, school].join('|'), 'utf8')
|
||
.digest('hex').substring(0, 12);
|
||
}
|
||
|
||
// ---------- 数据库 ----------
|
||
|
||
function initDB() {
|
||
const db = new Database(DB_PATH);
|
||
db.pragma('journal_mode = WAL');
|
||
db.exec(`
|
||
CREATE TABLE IF NOT EXISTS candidates (
|
||
id TEXT PRIMARY KEY, platform TEXT NOT NULL DEFAULT 'zhaopin',
|
||
name TEXT NOT NULL, age INTEGER, education TEXT, experience TEXT,
|
||
exp_years REAL, school TEXT, expected_city TEXT, expected_salary TEXT,
|
||
score INTEGER, score_level TEXT, role_tag TEXT,
|
||
greeted INTEGER NOT NULL DEFAULT 0, greeted_at TEXT,
|
||
raw_text TEXT, education_detail TEXT, work_detail TEXT, cert_detail TEXT,
|
||
card_info TEXT, score_details TEXT,
|
||
scraped_at TEXT, updated_at TEXT, notes TEXT DEFAULT ''
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_cand_level ON candidates(score_level);
|
||
CREATE INDEX IF NOT EXISTS idx_cand_greeted ON candidates(greeted);
|
||
CREATE TABLE IF NOT EXISTS greet_log (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
candidate_id TEXT NOT NULL, action TEXT NOT NULL, result TEXT,
|
||
created_at TEXT NOT NULL DEFAULT (datetime('now','localtime')));
|
||
CREATE TABLE IF NOT EXISTS scrape_log (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
mode TEXT, target_count INTEGER, new_count INTEGER, skip_count INTEGER,
|
||
greet_count INTEGER, error_count INTEGER,
|
||
created_at TEXT NOT NULL DEFAULT (datetime('now','localtime')));
|
||
`);
|
||
// 兼容 v9 旧库：补缺失列
|
||
const cols = new Set(db.prepare("PRAGMA table_info(candidates)").all().map(c => c.name));
|
||
for (const [col, ddl] of [['role_tag', "ALTER TABLE candidates ADD COLUMN role_tag TEXT"],
|
||
['account', "ALTER TABLE candidates ADD COLUMN account TEXT DEFAULT 'hr1'"],
|
||
['activity', "ALTER TABLE candidates ADD COLUMN activity TEXT"]]) {
|
||
if (!cols.has(col)) db.exec(ddl);
|
||
}
|
||
return db;
|
||
}
|
||
|
||
function upsertCandidate(db, d) {
|
||
const now = new Date().toISOString();
|
||
const prev = db.prepare('SELECT greeted, greeted_at FROM candidates WHERE id = ?').get(d.id);
|
||
const greeted = (prev && prev.greeted) || d.greeted ? 1 : 0;
|
||
db.prepare(`
|
||
INSERT INTO candidates (id, platform, account, name, age, education, experience, exp_years,
|
||
school, expected_city, expected_salary, activity, score, score_level, role_tag,
|
||
greeted, greeted_at, raw_text, education_detail, work_detail, cert_detail,
|
||
card_info, score_details, scraped_at, updated_at, notes)
|
||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||
ON CONFLICT(id) DO UPDATE SET
|
||
account=excluded.account, name=excluded.name, age=excluded.age,
|
||
education=excluded.education, experience=excluded.experience, exp_years=excluded.exp_years,
|
||
school=excluded.school, expected_city=excluded.expected_city,
|
||
expected_salary=excluded.expected_salary, activity=excluded.activity,
|
||
score=excluded.score, score_level=excluded.score_level, role_tag=excluded.role_tag,
|
||
greeted=excluded.greeted,
|
||
greeted_at=CASE WHEN excluded.greeted=1 AND candidates.greeted=0 THEN excluded.greeted_at ELSE candidates.greeted_at END,
|
||
raw_text=excluded.raw_text, education_detail=excluded.education_detail,
|
||
work_detail=excluded.work_detail, cert_detail=excluded.cert_detail,
|
||
card_info=excluded.card_info, score_details=excluded.score_details,
|
||
updated_at=excluded.updated_at, notes=excluded.notes
|
||
`).run(d.id, 'zhaopin', ACCOUNT, d.name, d.age, d.education, d.experience,
|
||
parseExperienceYears(d.experience || d.positionExp), d.school, d.expectedCity || '',
|
||
d.expectedSalary || '', d.activity || '', d.score ?? null, d.scoreLevel || '',
|
||
d.roleTag || '', greeted, d.greeted ? now : (prev ? prev.greeted_at : null),
|
||
d.rawText || '', d.educationDetail || '', d.workDetail || '', d.certDetail || '',
|
||
d.cardInfo ? JSON.stringify(d.cardInfo) : null,
|
||
d.scoreDetails ? JSON.stringify(d.scoreDetails) : null,
|
||
d.scrapedAt || now, now, d.notes || '');
|
||
}
|
||
|
||
// ---------- 页面操作 ----------
|
||
|
||
async function closeModal(page) {
|
||
await page.evaluate(() => {
|
||
document.querySelector('.new-shortcut-resume__close')?.click();
|
||
document.querySelector('.km-modal--open')?.classList.remove('km-modal--open');
|
||
});
|
||
await randomSleep(1, 2);
|
||
}
|
||
|
||
// 弹窗内打招呼（v9：天然绑定正确的人，杜绝列表 index 错位）
|
||
async function clickGreetInModal(page) {
|
||
// v10.10（2026-09-02）：原生 locator 点击（合成 click 对部分弹窗无效）
|
||
// + 轮询等待按钮变化（最长8秒，替代固定2.5秒）+ 失败重试1次
|
||
for (let attempt = 0; attempt < 2; attempt++) {
|
||
try {
|
||
const btn = page.locator('.new-shortcut-resume__modal button.km-button--primary.km-button--filled').first();
|
||
const t = (await btn.innerText({ timeout: 5000 }).catch(() => '')).trim();
|
||
if (t === '继续沟通') return true; // 已打过
|
||
if (!t.includes('打招呼')) return false;
|
||
await btn.click({ timeout: 5000 });
|
||
// 轮询等按钮变"继续沟通"（最长8秒）
|
||
for (let i = 0; i < 16; i++) {
|
||
await new Promise(r => setTimeout(r, 500));
|
||
const t2 = (await btn.innerText({ timeout: 2000 }).catch(() => '')).trim();
|
||
if (t2 === '继续沟通') return true;
|
||
}
|
||
// 8秒没变——重试一次（第二次 attempt）
|
||
} catch (_) { /* 进重试 */ }
|
||
}
|
||
return false;
|
||
}
|
||
|
||
// 人类行为模拟：弹窗内容超一屏时随机滚动
|
||
async function humanBrowseModal(page) {
|
||
await page.evaluate(async () => {
|
||
const wrap = document.querySelector('.new-resume-detail .km-scrollbar__wrap');
|
||
if (!wrap || wrap.scrollHeight <= wrap.clientHeight + 50) return;
|
||
const steps = 2 + Math.floor(Math.random() * 4);
|
||
for (let i = 0; i < steps; i++) {
|
||
wrap.scrollTop += 120 + Math.random() * 180;
|
||
await new Promise(r => setTimeout(r, 800 + Math.random() * 1200));
|
||
}
|
||
if (Math.random() < 0.5) wrap.scrollTop -= 60; // 随机回看
|
||
});
|
||
}
|
||
|
||
// 聊天剩余次数（.rights-chat-count .rights-count-number）
|
||
async function getChatCount(page) {
|
||
return page.evaluate(() => {
|
||
const el = document.querySelector('.rights-chat-count .rights-count-number');
|
||
const n = el ? parseInt(el.innerText.trim(), 10) : NaN;
|
||
return Number.isFinite(n) ? n : null;
|
||
}).catch(() => null);
|
||
}
|
||
|
||
// 卡片结构化提取（v9：class 选择器，不解析 innerText；含同事聊过/已打招呼标识）
|
||
async function getCardQuickInfo(page, dataIndex) {
|
||
return page.evaluate((di) => {
|
||
const card = document.querySelector(`.recommend-item[data-index="${di}"]`);
|
||
if (!card || card.offsetParent === null) return null;
|
||
const q = (sel) => card.querySelector(sel)?.innerText?.trim() || '';
|
||
const name = q('.talent-basic-info__name--inner');
|
||
const ageRaw = q('.age-label');
|
||
const expRaw = q('.work-years-label');
|
||
const chatText = card.querySelector('.cv-test-recommend-chat')?.innerText?.trim() || '';
|
||
return {
|
||
dataIndex: di, name,
|
||
age: parseInt((ageRaw.match(/\d+/) || [0])[0], 10) || null,
|
||
experience: expRaw, education: q('.education-label'),
|
||
careerStatus: q('.career-status-label'),
|
||
coworkerChatted: !!card.querySelector('.new-communication-record'),
|
||
greeted: chatText === '继续沟通',
|
||
activity: card.querySelector('.global-active-tag')?.innerText?.trim().replace(/\n/g, ' ') || '',
|
||
cardText: card.innerText || '', // v10.2：卡片全文用于列表级黑名单企业预筛（不打开弹窗）
|
||
};
|
||
}, dataIndex);
|
||
}
|
||
|
||
// v9 导航：nextElementSibling 找下一张卡（虚拟列表 data-index 会复用，兄弟关系才可靠）
|
||
async function navigateToNextCard(page, dataIndex) {
|
||
return page.evaluate((di) => {
|
||
const cur = document.querySelector(`.recommend-item[data-index="${di}"]`);
|
||
if (!cur) return null;
|
||
const item = cur.closest('[role="listitem"]');
|
||
const nextItem = item?.nextElementSibling;
|
||
const nextCard = nextItem?.querySelector('.recommend-item');
|
||
return nextCard ? nextCard.getAttribute('data-index') : null;
|
||
}, dataIndex);
|
||
}
|
||
|
||
// 弹窗数据提取（v5 逻辑沿用）
|
||
async function extractModalData(page) {
|
||
return page.evaluate(() => {
|
||
const modal = document.querySelector('.new-shortcut-resume__modal');
|
||
if (!modal) return null;
|
||
const text = modal.innerText || '';
|
||
function section(startPat, endPat) {
|
||
const sm = text.match(new RegExp(startPat));
|
||
if (!sm) return '';
|
||
const rest = text.substring(sm.index + sm[0].length);
|
||
const em = rest.match(new RegExp(endPat));
|
||
return (em ? rest.substring(0, em.index) : rest.substring(0, 500)).trim();
|
||
}
|
||
const nameMatch = text.match(/^([^\n]+)/);
|
||
const ageMatch = text.match(/(\d+)岁/);
|
||
const yearMatch = text.match(/\((\d{4})年(\d+)月\)/);
|
||
const eduMatch = text.match(/(博士|硕士|本科|大专|中专|高中|初中)/);
|
||
const expMatch = text.match(/\d+年经验|应届生|在校生/) ||
|
||
text.split('\n').map(l => l.trim()).find(l => /^\d{1,2}年$/.test(l) && (l = l.trim()));
|
||
const salaryMatch = text.match(/([\d.]+[-~到][\d.]+千?\/?月?|面议|薪资面议)/);
|
||
const cityMatch = text.match(/\[([^\]]+)\]/);
|
||
let school = '';
|
||
const eduSection = section('教育经历', '个人优势|专业技能|语言能力|证书|打电话|打招呼');
|
||
if (eduSection) {
|
||
for (const line of eduSection.split('\n').map(l => l.trim()).filter(Boolean)) {
|
||
if (!/\d{4}/.test(line) && !/^(本科|大专|硕士|博士|统招|专升本)$/.test(line) && line.length > 2 && line.length < 25) { school = line; break; }
|
||
}
|
||
}
|
||
let positionExp = '';
|
||
const pm = text.match(/岗位经验\s*\n\S+\s*\n(\d+年\d*个月?)/);
|
||
if (pm) positionExp = pm[1];
|
||
// 人才处理记录条数（v10.1：>2条说明已被多位同事处理过，不再打招呼）
|
||
let staffLogCount = null;
|
||
const logsCountEl = modal.querySelector('.new-resume-sidebar__logs-count');
|
||
if (logsCountEl) {
|
||
const cm = (logsCountEl.innerText || '').match(/\((\d+)条\)/);
|
||
if (cm) staffLogCount = parseInt(cm[1], 10);
|
||
}
|
||
// ===== 阻碍即停：弹窗有效性核查（v10.6 通用机制）=====
|
||
// 正常简历弹窗：有姓名+年龄/学历等结构信号、内容较长、含标准段落
|
||
// 异常弹窗（限额页/提示页/空数据）：短文案、无结构信号
|
||
// 触发条件：缺姓名 或 缺全部结构信号（年龄/学历/经验/教育段/工作段）——即"这不是一份简历"
|
||
const hasName = !!(nameMatch && nameMatch[1] && nameMatch[1].trim() && !/提示|通知|警告|限额|上限/.test(nameMatch[1]));
|
||
const hasStructure = !!(ageMatch || eduMatch || expMatch || eduSection ||
|
||
section('工作经历', '教育经历|个人优势|专业技能|语言能力|证书|打电话|打招呼'));
|
||
// v10.8：候选人级提示 ≠ 平台阻碍。"对方已关闭求职/简历已删除/对方已找到工作/用户已隐藏在线简历"等
|
||
// 是单个候选人的状态提示，应跳过该人继续；只有限额/系统级文案才是平台阻碍
|
||
// v10.9（2026-09-02）：补"用户已隐藏在线简历/暂不可查看/已屏蔽"——同样是被关闭的单人弹窗
|
||
// v10.11（2026-09-15）：补"当前职位未上线"——职位级状态（推荐位职位被下线，所有弹窗同文案），终态停止
|
||
const body = text;
|
||
const jobLevelNotice = /当前职位未上线|职位已下线|职位已关闭/.test(body) && body.length < 200;
|
||
const candidateLevelNotice = /已关闭求职|无法继续进行沟通|已删除|简历不存在|已找到工作|职位已关闭|隐藏在线简历|暂不可查看|已屏蔽|不合适该职位|已停止找|违规行为|系统已为您自动屏蔽/.test(body) && body.length < 250;
|
||
const modalBlocked = (!hasName || !hasStructure) && !candidateLevelNotice && !jobLevelNotice;
|
||
return {
|
||
modalBlocked,
|
||
candidateLevelNotice,
|
||
staffLogCount,
|
||
name: nameMatch ? nameMatch[1].trim() : '',
|
||
age: ageMatch ? parseInt(ageMatch[1], 10) : null,
|
||
birthYear: yearMatch ? parseInt(yearMatch[1], 10) : null,
|
||
education: eduMatch ? eduMatch[1] : '',
|
||
experience: expMatch ? String(expMatch[0] || expMatch) : (positionExp || ''),
|
||
positionExp, expectedSalary: salaryMatch ? salaryMatch[1] : '',
|
||
expectedCity: cityMatch ? cityMatch[1] : '', school, rawText: text,
|
||
educationDetail: eduSection,
|
||
workDetail: section('工作经历', '教育经历|个人优势|专业技能|语言能力|证书|打电话|打招呼'),
|
||
certDetail: section('所获证书|证书|语言能力', '打电话|打招呼|求职期望|教育经历|工作经历'),
|
||
scrapedAt: new Date().toISOString(),
|
||
};
|
||
});
|
||
}
|
||
|
||
// 从弹窗 URL 提取平台原生简历编号（v9：唯一可靠的跨批次去重标识）
|
||
function extractResumeNumber(page) {
|
||
const m = (page.url().match(/resumeNumber=([^&]+)/) || [])[1];
|
||
return m ? decodeURIComponent(m) : null;
|
||
}
|
||
|
||
// 公司黑名单：从工作经历提取公司名行
|
||
function extractCompanies(workDetail) {
|
||
return (workDetail || '').split('\n').map(l => l.trim())
|
||
.filter(l => l.length > 3 && l.length < 30 && !/\d{4}|^\d|[（(]|^职位|^岗位/.test(l));
|
||
}
|
||
|
||
// 停止旗标（v10.3：随时叫停机制——stop-tasks.sh 创建旗标文件，脚本每轮检查优雅退出）
|
||
const STOP_FLAG = '/opt/data/logs/recruitment/state/STOP.flag';
|
||
function stopRequested() { try { return require('fs').existsSync(STOP_FLAG); } catch { return false; } }
|
||
|
||
// 健康检测（v5 沿用）
|
||
async function checkPageHealth(page) {
|
||
const url = page.url();
|
||
if (url.includes('login') || url.includes('passport')) return '🚨 ALERT: 登录已失效，请重新登录后继续';
|
||
const info = await page.evaluate(() => {
|
||
const body = document.body?.innerText || '';
|
||
return {
|
||
captcha: !!document.querySelector('.verify-wrap,.captcha,.slider-verify,.tcaptcha-iframe') || /安全验证|请完成验证|滑块验证|图形验证码/i.test(body),
|
||
risk: /操作过于频繁|访问受限|账号异常|请求异常/i.test(body),
|
||
greetLimit: /打招呼已达上限|已达到.*上限|今日打招呼次数已用完|打招呼额度不足/i.test(body),
|
||
// 被顶号检测（2026-08-27）：km-modal 消息弹窗 + 精确文案 + 重新登录按钮，三者俱全才判定，防误报
|
||
kicked: !!document.querySelector('.km-modal.km-modal--message-box .km-modal__title-inner')
|
||
&& /账号已在其他设备或浏览器登录/.test(body)
|
||
&& !!document.querySelector('.km-modal.km-modal--message-box .km-button--primary'),
|
||
};
|
||
}).catch(() => ({ captcha: false, risk: false, greetLimit: false, kicked: false }));
|
||
if (info.kicked) return '🚨 ALERT: 账号被顶:您的账号已在其他设备或浏览器登录';
|
||
if (info.captcha) return '🚨 ALERT: 触发验证码/风控，已暂停';
|
||
if (info.risk) return '🚨 ALERT: 触发风控限制，已暂停';
|
||
if (info.greetLimit) return '🚨 ALERT: 打招呼已达上限，停止打招呼';
|
||
return null;
|
||
}
|
||
|
||
// ---------- 主流程 ----------
|
||
|
||
async function scrapeZhaopin(maxCount = 20, fullMode = false, scoreOnly = false) {
|
||
console.log(`[zhaopin v10] account=${ACCOUNT} cdp=${CDP_PORT} db=${DB_PATH}`);
|
||
console.log(`门槛配置: 岗位词${CONFIG.tradeKeywords.length}个 | 专业词${CONFIG.majorKeywords.length}个 | 证书词${CONFIG.certKeywords.length}个 | 毕业${CONFIG.gradYear.min}-${CONFIG.gradYear.max} | 年龄≤${CONFIG.maxAge}`);
|
||
|
||
const res = await fetch(`http://${CDP_HOST}:${CDP_PORT}/json/version`);
|
||
const info = await res.json();
|
||
const browser = await chromium.connectOverCDP(info.webSocketDebuggerUrl);
|
||
const ctx = browser.contexts()[0];
|
||
const page = ctx.pages().find(p => p.url().includes('recommend'));
|
||
if (!page) { console.log('ERROR: 未找到推荐页面（需在浏览器中打开 recommend 页）'); await browser.close(); process.exit(1); }
|
||
|
||
const db = initDB();
|
||
loadPersonBlacklist(db); // v10.13：个人黑名单
|
||
const existingIds = new Set(db.prepare('SELECT id FROM candidates').all().map(r => r.id));
|
||
// 启动快速跳过用的卡片指纹（name|age|education|experience，来自历史 card_info）
|
||
const storedCards = new Set(db.prepare("SELECT card_info FROM candidates WHERE card_info IS NOT NULL").all()
|
||
.map(r => { try { const c = JSON.parse(r.card_info); return [c.name, c.age, c.education, c.experience].join('|'); } catch (_) { return null; } })
|
||
.filter(Boolean));
|
||
|
||
let greeted = 0, errors = 0, newCount = 0, skipped = 0, irrelevant = 0;
|
||
let consecutiveErrors = 0, consecutiveGreetFails = 0, greetLimitReached = scoreOnly;
|
||
let currentDataIndex = null, processed = new Set(), noCardCount = 0;
|
||
let sessionProcessed = parseInt(process.env.SESSION_OFFSET || '0', 10) || 0, lastReloadAt = sessionProcessed; // v10.4：账号累计处理数（runner跨批传入）与上次刷新点
|
||
const RELOAD_EVERY = parseInt(process.env.RELOAD_EVERY || '200', 10); // v10.4：每累计200人刷新页面
|
||
|
||
const log = (m) => console.log(` ${m}`);
|
||
const stop = async (reason, code = 1) => {
|
||
console.log(reason);
|
||
db.prepare('INSERT INTO scrape_log (mode,target_count,new_count,skip_count,greet_count,error_count) VALUES (?,?,?,?,?,?)')
|
||
.run(`${fullMode ? 'full' : 'incr'}${scoreOnly ? '-score' : ''}`, maxCount, newCount, skipped, greeted, errors);
|
||
db.close(); await browser.close(); process.exit(code);
|
||
};
|
||
|
||
console.log(`已存储 ${existingIds.size} 人，${fullMode ? '全量' : '增量'}模式${scoreOnly ? '（仅评分）' : ''}`);
|
||
let chatLeft = await getChatCount(page);
|
||
if (chatLeft !== null) console.log(`聊天剩余: ${chatLeft} 次`);
|
||
|
||
// 启动快速跳过：定位第一个待处理卡片
|
||
let firstIndex = await page.evaluate(() => document.querySelector('.recommend-item')?.getAttribute('data-index'));
|
||
let skipN = 0;
|
||
while (firstIndex !== null && skipN < MAX_START_SKIP) {
|
||
const c = await getCardQuickInfo(page, firstIndex);
|
||
if (!c || !c.name) break;
|
||
const cardFp = [c.name, c.age, c.education, c.experience].join('|');
|
||
if (c.coworkerChatted || c.greeted || (c.age && c.age > CONFIG.maxAge) || (!fullMode && storedCards.has(cardFp))) {
|
||
firstIndex = await navigateToNextCard(page, firstIndex); skipN++;
|
||
continue;
|
||
}
|
||
break;
|
||
}
|
||
currentDataIndex = firstIndex;
|
||
console.log(`启动跳过 ${skipN} 人，从 data-index=${currentDataIndex} 开始`);
|
||
|
||
while (newCount < maxCount) {
|
||
// v10.7：批内 18 点软截止——处理完当前候选人就优雅收（STOP.flag 同款退出路径，日志带统计）
|
||
// 北京18:00 = UTC 10:00。之后不再开新弹窗，保证 18 点平稳落地，收队任务无需强杀
|
||
if (process.env.DAILY_CUTOFF !== '0' && new Date(Date.now() + 8 * 3600 * 1000).getUTCHours() >= 18) {
|
||
return stop('⏰ 到达北京时间18:00，批内软截止收工', 0);
|
||
}
|
||
// v10.3：随时叫停——每轮循环检查停止旗标（当前候选人处理完才退，不会打断打招呼中途）
|
||
if (stopRequested()) return stop('🛑 收到停止指令，优雅退出本批', 0);
|
||
// v10.4：账号累计处理满200人 → 刷新页面（清虚拟列表DOM累积/内存，重置滚动状态）
|
||
// 累计基数由 runner 通过 SESSION_OFFSET 传入（跨批次累计），批内用 offset+newCount 判断
|
||
if (sessionProcessed - lastReloadAt >= RELOAD_EVERY) {
|
||
console.log(` ♻️ 账号累计处理 ${sessionProcessed} 人（达${RELOAD_EVERY}人阈值），刷新页面...`);
|
||
await page.reload({ waitUntil: 'domcontentloaded', timeout: 60000 }).catch(e => console.log(`刷新失败: ${e.message}`));
|
||
await randomSleep(5, 10);
|
||
lastReloadAt = sessionProcessed;
|
||
processed = new Set();
|
||
currentDataIndex = null;
|
||
}
|
||
const alert = await checkPageHealth(page);
|
||
if (alert) {
|
||
if (alert.includes('上限') && !alert.includes('失效')) { console.log(alert); greetLimitReached = true; }
|
||
else return stop(alert);
|
||
}
|
||
if (await page.evaluate(() => !!document.querySelector('.km-modal--open'))) await closeModal(page);
|
||
|
||
if (currentDataIndex === null || processed.has(currentDataIndex)) {
|
||
await page.evaluate(() => window.scrollBy(0, window.innerHeight * 0.8));
|
||
await randomSleep(3, 5);
|
||
// v10.5：触底检测——出现"已经到底啦"指示器说明本批数据加载完，直接刷新页面拿新数据（不等累计人数阈值）
|
||
const hitBottom = await page.evaluate(() => !!document.querySelector('.recommend-indicator') &&
|
||
/已经到底/.test(document.querySelector('.recommend-indicator')?.innerText || ''));
|
||
if (hitBottom) {
|
||
console.log(' ⤓ 页面已触底（本批数据加载完），刷新页面获取新数据...');
|
||
await page.reload({ waitUntil: 'domcontentloaded', timeout: 60000 }).catch(e => console.log(`刷新失败: ${e.message}`));
|
||
await randomSleep(5, 10);
|
||
processed = new Set();
|
||
noCardCount = 0;
|
||
currentDataIndex = null; // 重新从头扫描（启动跳过逻辑会略过已处理的卡片）
|
||
continue;
|
||
}
|
||
const idx = await page.evaluate(() => document.querySelector('.recommend-item')?.getAttribute('data-index'));
|
||
if (idx === null && ++noCardCount >= 5) { console.log('连续无新卡片，结束'); break; }
|
||
currentDataIndex = idx; continue;
|
||
}
|
||
noCardCount = 0;
|
||
|
||
const card = await getCardQuickInfo(page, currentDataIndex);
|
||
if (!card || !card.name) { currentDataIndex = await navigateToNextCard(page, currentDataIndex); continue; }
|
||
processed.add(currentDataIndex);
|
||
|
||
if (card.coworkerChatted) { log(`[跳过-同事聊过] ${card.name}`); skipped++; currentDataIndex = await navigateToNextCard(page, currentDataIndex); continue; }
|
||
if (card.greeted) { log(`[跳过-已沟通] ${card.name}`); skipped++; currentDataIndex = await navigateToNextCard(page, currentDataIndex); continue; }
|
||
if (card.age && card.age > CONFIG.maxAge) { log(`[跳过-年龄>35] ${card.name} ${card.age}岁`); skipped++; currentDataIndex = await navigateToNextCard(page, currentDataIndex); continue; }
|
||
// v10.2：列表级黑名单企业预筛——卡片文本命中直接跳过，不打开弹窗
|
||
const cardBlackHit = COMPANY_BLACKLIST.find(b => card.cardText && card.cardText.includes(b));
|
||
if (cardBlackHit) { log(`🚫 [跳过-列表黑名单] ${card.name} | 卡片含"${cardBlackHit}"`); skipped++; currentDataIndex = await navigateToNextCard(page, currentDataIndex); continue; }
|
||
|
||
// v10.13：个人黑名单——弹窗级匹配（全名/resumeNumber/硬字段组），基于真实数据，卡片级不预筛
|
||
|
||
try {
|
||
// 打开弹窗（v10.12：Playwright原生点击替代evaluate合成click——平台反自动化检测会
|
||
// 对合成点击返回空白弹窗；原生点击走真实鼠标事件流 CDP Input.dispatchMouseEvent）
|
||
try {
|
||
const cardLoc = page.locator(`.recommend-item[data-index="${currentDataIndex}"] .resume-item__content`).first();
|
||
await cardLoc.click({ timeout: 8000 });
|
||
} catch (_) {
|
||
// 兜底：内容区选择器失败时点卡片本身（同样用原生点击）
|
||
await page.locator(`.recommend-item[data-index="${currentDataIndex}"]`).first().click({ timeout: 8000 });
|
||
}
|
||
await randomSleep(3, 5);
|
||
// 弹窗有效性双检：不仅看"有弹窗"，还要看是简历弹窗（空壳km-modal说明被拦截）
|
||
const modalOk = await page.evaluate(() => ({
|
||
open: !!document.querySelector('.km-modal--open, .km-modal__backdrop--active'),
|
||
isResume: !!document.querySelector('.new-shortcut-resume__modal'),
|
||
}));
|
||
if (!modalOk.open) {
|
||
log('弹窗未打开，跳过'); errors++; currentDataIndex = await navigateToNextCard(page, currentDataIndex); continue;
|
||
}
|
||
if (!modalOk.isResume) {
|
||
// 空壳弹窗：关掉重试一次原生点击；再不行跳过该人
|
||
log('弹窗为空壳（非简历弹窗），关闭重试');
|
||
await closeModal(page);
|
||
await randomSleep(2, 3);
|
||
try {
|
||
await page.locator(`.recommend-item[data-index="${currentDataIndex}"] .resume-item__content`).first().click({ timeout: 8000 });
|
||
await randomSleep(3, 5);
|
||
} catch (_) { }
|
||
const retryOk = await page.evaluate(() => !!document.querySelector('.new-shortcut-resume__modal'));
|
||
if (!retryOk) {
|
||
await closeModal(page);
|
||
log('重试仍是空壳弹窗，跳过该卡片'); errors++; currentDataIndex = await navigateToNextCard(page, currentDataIndex); continue;
|
||
}
|
||
}
|
||
|
||
// 平台唯一标识
|
||
const resumeNumber = extractResumeNumber(page);
|
||
if (resumeNumber && !fullMode && existingIds.has(resumeNumber)) {
|
||
await closeModal(page);
|
||
log(`[跳过-已存储] ${card.name}`); skipped++;
|
||
currentDataIndex = await navigateToNextCard(page, currentDataIndex); continue;
|
||
}
|
||
|
||
// 刷新检测：弹窗人名 vs 卡片人名
|
||
const refreshed = await page.evaluate((expected) => {
|
||
const actual = document.querySelector('.new-resume-detail .resume-name')?.innerText?.trim() || '';
|
||
return expected && actual && !actual.includes(expected.replace(/(先生|女士)$/, ''));
|
||
}, card.name);
|
||
if (refreshed) { await closeModal(page); log('页面已刷新（列表变化），重扫'); currentDataIndex = null; continue; }
|
||
|
||
const data = await extractModalData(page);
|
||
// v10.13：个人黑名单——弹窗真实数据匹配（全名/resumeNumber/硬字段组全命中）
|
||
const pHit = personBlackHitModal(data?.name, resumeNumber, data?.rawText || '');
|
||
if (pHit) {
|
||
await closeModal(page);
|
||
log(`🚫 [跳过-个人黑名单] ${card.name} | 命中: ${pHit}`); skipped++;
|
||
currentDataIndex = await navigateToNextCard(page, currentDataIndex); continue;
|
||
}
|
||
// v10.8：候选人级提示（对方已关闭求职等）→ 跳过该人继续，不是平台阻碍
|
||
if (data?.candidateLevelNotice && !data?.name) {
|
||
log(`⊘ 候选人状态提示，跳过: ${(data.rawText || '').split('\n').filter(Boolean)[0] || '(空)'}`);
|
||
await closeModal(page);
|
||
skipped++;
|
||
currentDataIndex = await navigateToNextCard(page, currentDataIndex);
|
||
continue;
|
||
}
|
||
// v10.6 阻碍即停：弹窗不是有效简历（限额/提示页/空数据）→ 不入库，立即熔断
|
||
if (data?.modalBlocked) {
|
||
const snippet = (data.rawText || '').split('\n').filter(Boolean).slice(0, 3).join(' / ').slice(0, 120);
|
||
await closeModal(page);
|
||
return stop(`🚨 ALERT: 弹窗无有效简历数据（可能触发平台限制），弹窗内容: ${snippet || '(空)'}`);
|
||
}
|
||
if (!data?.name) { await closeModal(page); log('数据提取失败'); errors++; currentDataIndex = await navigateToNextCard(page, currentDataIndex); continue; }
|
||
|
||
await humanBrowseModal(page);
|
||
|
||
// 公司黑名单
|
||
const companies = extractCompanies(data.workDetail || '');
|
||
const blackHit = COMPANY_BLACKLIST.find(b => companies.some(c => c.includes(b)));
|
||
if (blackHit) {
|
||
await closeModal(page);
|
||
log(`🚫 黑名单命中: ${data.name} | "${blackHit}"`); skipped++;
|
||
currentDataIndex = await navigateToNextCard(page, currentDataIndex); continue;
|
||
}
|
||
|
||
// ===== 硬性预筛 + 评分（v10 核心） =====
|
||
data.id = resumeNumber || fingerprint(data.name, data.age, data.education, data.experience, data.school);
|
||
data.platform = 'zhaopin';
|
||
data.activity = card.activity || '';
|
||
data.greeted = card.greeted;
|
||
data.cardInfo = { name: card.name, age: card.age, education: card.education, experience: card.experience, activity: card.activity, dataIndex: currentDataIndex };
|
||
|
||
const result = evaluateCandidate(data);
|
||
data.score = result.score;
|
||
data.scoreLevel = result.level;
|
||
data.roleTag = result.roleTag;
|
||
data.scoreDetails = result.gate.pass ? { ...result.details, gateReasons: result.gate.reasons } : { gateReason: result.gate.reason };
|
||
data.notes = result.gate.pass ? result.gate.reasons.join(';') : result.gate.reason;
|
||
|
||
upsertCandidate(db, data);
|
||
existingIds.add(data.id);
|
||
newCount++;
|
||
sessionProcessed++; // v10.4：累计处理数（供200人刷新判断）
|
||
|
||
if (!result.gate.pass) {
|
||
irrelevant++;
|
||
log(`⛔ 不相关 ${data.name} | ${data.age}岁 | ${result.gate.reason} | 不评分不打招呼`);
|
||
} else {
|
||
const icon = result.level === '推荐' ? '✅' : result.level === '待定' ? '⏳' : '❌';
|
||
log(`${icon} ${data.name} | ${result.total}分 ${result.level} [${result.roleTag}] | ${result.gate.reasons.filter(r => r !== 'year-unknown').join(';')}`);
|
||
}
|
||
|
||
// ===== 弹窗内打招呼（仅 推荐/待定 且门槛通过 且 人才处理记录≤2条） =====
|
||
if (!greetLimitReached && result.gate.pass && (result.level === '推荐' || result.level === '待定') && !data.greeted
|
||
&& !(data.staffLogCount !== null && data.staffLogCount > 2)) {
|
||
const ok = await clickGreetInModal(page);
|
||
if (ok) {
|
||
consecutiveGreetFails = 0; greeted++;
|
||
data.greeted = true;
|
||
upsertCandidate(db, data);
|
||
db.prepare('INSERT INTO greet_log (candidate_id, action, result) VALUES (?,?,?)').run(data.id, 'greet', 'success');
|
||
log(`✋ 已打招呼: ${data.name}`);
|
||
chatLeft = await getChatCount(page);
|
||
if (chatLeft !== null && chatLeft <= 5) console.log(`⚠ 聊天剩余 ${chatLeft} 次`);
|
||
if (chatLeft !== null && chatLeft <= 0) return stop('🚨 聊天次数耗尽，结束本批', 0);
|
||
} else {
|
||
consecutiveGreetFails++;
|
||
db.prepare('INSERT INTO greet_log (candidate_id, action, result) VALUES (?,?,?)').run(data.id, 'greet', 'failed');
|
||
log(`⚠ 打招呼失败（连续${consecutiveGreetFails}次）`);
|
||
if (consecutiveGreetFails >= 3) return stop('🚨 连续3次打招呼失败，可能风控/上限');
|
||
}
|
||
} else if (result.gate.pass && (result.level === '推荐' || result.level === '待定') && !data.greeted
|
||
&& data.staffLogCount !== null && data.staffLogCount > 2) {
|
||
log(`⊘ 人才处理记录过多(${data.staffLogCount}条): ${data.name} | 通过筛选但不打招呼`);
|
||
}
|
||
|
||
await closeModal(page);
|
||
if (newCount % 10 === 0) { console.log(` --- 已处理 ${newCount} 人（不相关 ${irrelevant}），休息 ---`); await randomSleep(8, 15); }
|
||
await randomSleep(2, 4);
|
||
consecutiveErrors = 0;
|
||
currentDataIndex = await navigateToNextCard(page, currentDataIndex);
|
||
} catch (e) {
|
||
errors++; consecutiveErrors++;
|
||
log(`错误: ${String(e.message).substring(0, 150)}`);
|
||
await closeModal(page).catch(() => {});
|
||
await randomSleep(2, 4);
|
||
if (consecutiveErrors >= 5) return stop('🚨 连续5次操作错误，停止');
|
||
currentDataIndex = await navigateToNextCard(page, currentDataIndex);
|
||
}
|
||
}
|
||
|
||
console.log(`\n${'='.repeat(50)}\n完成: 新增 ${newCount}（其中不相关拦截 ${irrelevant}）| 跳过 ${skipped} | 打招呼 ${greeted} | 错误 ${errors}\n数据库总计: ${db.prepare('SELECT COUNT(*) c FROM candidates').get().c} 人`);
|
||
db.prepare('INSERT INTO scrape_log (mode,target_count,new_count,skip_count,greet_count,error_count) VALUES (?,?,?,?,?,?)')
|
||
.run(`${fullMode ? 'full' : 'incr'}${scoreOnly ? '-score' : ''}`, maxCount, newCount, skipped, greeted, errors);
|
||
db.close();
|
||
await browser.close();
|
||
}
|
||
|
||
// ---------- 入口 ----------
|
||
|
||
const args = process.argv.slice(2);
|
||
scrapeZhaopin(
|
||
parseInt(args.filter(a => !a.startsWith('--'))[0], 10) || 20,
|
||
args.includes('--full'),
|
||
args.includes('--score-only'),
|
||
).catch(e => { console.error('FATAL:', e.message); process.exit(1); });
|