hermes
138 lines
7.0 KiB
JavaScript
138 lines
7.0 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* 18:00 强制收队 + 异常检测 v1.0（2026-08-26）
|
||
* 由 cron 在北京时间 18:05 触发（UTC 10:05）。
|
||
*
|
||
* 职责：
|
||
* 1. 强杀所有仍在运行的 daily-runner / zhaopin-scrape 进程（正常应已自行结束）
|
||
* 2. 异常检测：
|
||
* a. 18点仍在跑的进程 → 判定"未按时收队"（含失联/卡死）
|
||
* b. 卡死检测：进程 cputime 为 0 或远小于 etime（pitfalls #81）
|
||
* c. 状态文件检查：state-hrX.json 缺失/非当日 → 该账号今日未正常运行
|
||
* 3. 若日报今日未发（runner 异常没发成）→ 补发生成日报到群
|
||
* 4. 输出检测结果（cron agent 转发给用户私聊）
|
||
*/
|
||
const { execSync } = require('child_process');
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
|
||
const SCRIPTS = '/opt/data/skills/recruitment-resume-screener/scripts';
|
||
const STATE_DIR = '/opt/data/logs/recruitment/state';
|
||
fs.mkdirSync(STATE_DIR, { recursive: true });
|
||
const GROUP = 'oc_7ef4980f25ec2a80bb013d5dd79b9864'; // 群：每日日报（2026-08-27 修正回群，消息策略定稿）
|
||
const HERMES = '/opt/hermes/.venv/bin/hermes';
|
||
const bjNow = () => new Date(Date.now() + 8 * 3600 * 1000);
|
||
const today = () => bjNow().toISOString().slice(0, 10);
|
||
const log = (m) => console.log(`[enforce] ${m}`);
|
||
|
||
const findings = []; // 异常发现
|
||
let reportSent = false;
|
||
|
||
// ---------- 1. 进程检查与强杀 ----------
|
||
let psOut = '';
|
||
try { psOut = execSync("ps aux | grep -E 'daily-runner|zhaopin-scrape' | grep -v grep || true", { encoding: 'utf8' }); } catch { }
|
||
|
||
const lines = psOut.split('\n').filter(l => l.trim());
|
||
if (lines.length === 0) {
|
||
log('无残留进程 ✅');
|
||
} else {
|
||
for (const line of lines) {
|
||
const pid = line.trim().split(/\s+/)[1];
|
||
// 卡死检测：cputime vs etime
|
||
let hung = false, detail = '';
|
||
try {
|
||
const pi = execSync(`ps -p ${pid} -o pid,stat,etime,cputime,rss,cmd --no-headers`, { encoding: 'utf8' }).trim();
|
||
detail = pi;
|
||
const parts = pi.split(/\s+/);
|
||
const etime = parts[2] || '', cputime = parts[3] || '';
|
||
const toSec = (t) => { // etime: [[dd-]hh:]mm:ss
|
||
let s = 0;
|
||
const m = t.match(/(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+)/);
|
||
if (m) s = (+m[1] || 0) * 86400 + (+m[2] || 0) * 3600 + (+m[3]) * 60 + (+m[4]);
|
||
return s;
|
||
};
|
||
const cSec = toSec(cputime), eSec = toSec(etime);
|
||
if (cSec === 0 || (eSec > 600 && cSec / eSec < 0.01)) hung = true;
|
||
} catch { }
|
||
findings.push(`18点后仍在运行${hung ? '（疑似卡死）' : ''}: ${line.trim().slice(0, 140)}`);
|
||
try { process.kill(parseInt(pid, 10), 'SIGKILL'); log(`已强杀 PID ${pid}`); }
|
||
catch (e) { log(`杀 ${pid} 失败: ${e.message}`); findings.push(`强杀失败 PID ${pid}: ${e.message}`); }
|
||
}
|
||
// 复查
|
||
try {
|
||
const left = execSync("ps aux | grep -E 'daily-runner|zhaopin-scrape' | grep -v grep || true", { encoding: 'utf8' }).trim();
|
||
if (left) findings.push(`强杀后仍有残留：\n${left.slice(0, 300)}`);
|
||
} catch { }
|
||
}
|
||
|
||
// ---------- 2. 状态文件检查 ----------
|
||
for (const acct of ['hr1', 'hr2']) {
|
||
const f = path.join(STATE_DIR, `state-${acct}.json`);
|
||
try {
|
||
const s = JSON.parse(fs.readFileSync(f, 'utf8'));
|
||
if (s.date !== today()) findings.push(`${acct}: 今日无状态记录（runner 未启动或启动即失败）`);
|
||
else log(`${acct}: 今日状态=${s.status} endedAt=${s.endedAt}`);
|
||
} catch {
|
||
findings.push(`${acct}: 状态文件缺失（runner 未启动）`);
|
||
}
|
||
}
|
||
|
||
// ---------- 3. 日报发送（v1.1：18:05 无条件生成当日完整数据日报发群）----------
|
||
// 2026-08-27 教训：白天异常时旧代码发过一次不完整日报并落 flag，收队看到 flag 就跳过，
|
||
// 导致当天最终日报缺失。现在收队是日报唯一出口，无条件以最新数据重发。
|
||
const sentFlag = path.join(STATE_DIR, `report-sent-${today()}`);
|
||
{
|
||
log('生成当日日报并发群...');
|
||
try {
|
||
const env = { ...process.env };
|
||
// 从状态文件取聊天剩余 + 状态文案
|
||
for (const [acct, key, skey] of [['hr1', 'CHAT_LEFT_HR1', 'HR1_STATUS'], ['hr2', 'CHAT_LEFT_HR2', 'HR2_STATUS']]) {
|
||
try {
|
||
const s = JSON.parse(fs.readFileSync(path.join(STATE_DIR, `state-${acct}.json`), 'utf8'));
|
||
if (s.chatLeft != null) env[key] = String(s.chatLeft);
|
||
const map = {
|
||
kicked_out: '🚨 被顶号(他处登录)', login_expired: '🚨 登录失效', platform_blocked: '⛔ 平台阻碍(限额等)',
|
||
qr_expired: '🚨 等扫码但二维码过期', waiting_relogin: '⏳ 等待扫码登录', stopped_by_user: '⏹️ 手动停止',
|
||
tunnel_failed: '⚠️ 隧道失败', alert: '⚠️ 异常中止', chat_exhausted: '✅ 聊天耗尽收工',
|
||
time_cutoff: '✅ 18点收工', running: '✅ 在线',
|
||
};
|
||
if (s.date === today()) env[skey] = map[s.status] || '✅ 在线';
|
||
} catch { }
|
||
}
|
||
// v1.3（2026-09-15 用户定稿）：按账号分开发送，每个开工账号一条消息（不再合并合集）
|
||
const { execSync: es } = require('child_process');
|
||
// 先探一次全量渲染拿开工账号列表（输出空=都没开工）
|
||
const probe = es(`node ${SCRIPTS}/daily-report.js`, { cwd: SCRIPTS, encoding: 'utf8', timeout: 60000, env });
|
||
if (!probe.trim()) {
|
||
log('双账号今日均无数据，日报不发送（不发空消息）');
|
||
fs.writeFileSync(sentFlag, new Date().toISOString());
|
||
reportSent = 'skipped_empty';
|
||
} else {
|
||
// 判断开工账号：有数据的才发
|
||
for (const [acct, envKeys] of [['hr1', { left: 'CHAT_LEFT_HR1', st: 'HR1_STATUS' }], ['hr2', { left: 'CHAT_LEFT_HR2', st: 'HR2_STATUS' }]]) {
|
||
const envA = { ...env };
|
||
try {
|
||
const s = JSON.parse(fs.readFileSync(path.join(STATE_DIR, `state-${acct}.json`), 'utf8'));
|
||
if (s.chatLeft != null) envA[envKeys.left] = String(s.chatLeft);
|
||
if (s.date === today()) envA[envKeys.st] = s.status === 'time_cutoff' ? '✅' : (s.status === 'running' ? '✅' : '⚠️' + s.status);
|
||
} catch { }
|
||
const mdA = es(`node ${SCRIPTS}/daily-report.js --account ${acct}`, { cwd: SCRIPTS, encoding: 'utf8', timeout: 60000, env: envA });
|
||
if (!mdA.trim()) continue; // 该账号无数据，跳过不发
|
||
fs.writeFileSync(`/tmp/daily-report-${acct}-${today()}.md`, mdA);
|
||
es(`${HERMES} send -t "feishu:${GROUP}" -f /tmp/daily-report-${acct}-${today()}.md`, { stdio: ['pipe', 'pipe', 'pipe'], timeout: 60000 });
|
||
log(`${acct} 日报已单独发送到群 ✅`);
|
||
}
|
||
fs.writeFileSync(sentFlag, new Date().toISOString());
|
||
reportSent = true;
|
||
}
|
||
} catch (e) { findings.push(`日报发送失败: ${String(e.message).slice(0, 150)}`); }
|
||
}
|
||
|
||
// ---------- 4. 汇总输出 ----------
|
||
console.log('\n========== 18点收队结果 ==========');
|
||
console.log(`日报: ${reportSent ? '✅ 已发送' : '❌ 未发送'}`);
|
||
console.log(`异常: ${findings.length === 0 ? '无 ✅' : findings.length + ' 项 ⚠️'}`);
|
||
for (const f of findings) console.log(` ⚠️ ${f}`);
|
||
console.log('==================================');
|
||
process.exit(0);
|