109 lines
4.9 KiB
JavaScript
109 lines
4.9 KiB
JavaScript
|
#!/usr/bin/env node
|
|||
|
|
/**
|
|||
|
|
* 18:00 强制收队 + 异常检测 v1.0（2026-08-26）
|
|||
|
|
* 由 cron 在北京时间 18:05 触发（UTC 10:05）。
|
|||
|
|
*
|
|||
|
|
* 职责：
|
|||
|
|
* 1. 强杀所有仍在运行的 daily-runner / zhaopin-scrape 进程（正常应已自行结束）
|
|||
|
|
* 2. 异常检测：
|
|||
|
|
* a. 18点仍在跑的进程 → 判定"未按时收队"（含失联/卡死）
|
|||
|
|
* b. 卡死检测：进程 cputime 为 0 或远小于 etime（pitfalls #81）
|
|||
|
|
* c. 状态文件检查：state-hrX.json 缺失/非当日 → 该账号今日未正常运行
|
|||
|
|
* 3. 若日报今日未发（runner 异常没发成）→ 补发生成日报到群
|
|||
|
|
* 4. 输出检测结果（cron agent 转发给用户私聊）
|
|||
|
|
*/
|
|||
|
|
const { execSync } = require('child_process');
|
|||
|
|
const fs = require('fs');
|
|||
|
|
const path = require('path');
|
|||
|
|
|
|||
|
|
const SCRIPTS = '/opt/data/skills/recruitment-resume-screener/scripts';
|
|||
|
|
const STATE_DIR = '/opt/data/logs/recruitment/state';
|
|||
|
|
fs.mkdirSync(STATE_DIR, { recursive: true });
|
|||
|
|
const GROUP = 'oc_d09c77b62ca5f90f6699830fbc695c44'; // 2026-08-26 修正:只发私聊
|
|||
|
|
const HERMES = '/opt/hermes/.venv/bin/hermes';
|
|||
|
|
const bjNow = () => new Date(Date.now() + 8 * 3600 * 1000);
|
|||
|
|
const today = () => bjNow().toISOString().slice(0, 10);
|
|||
|
|
const log = (m) => console.log(`[enforce] ${m}`);
|
|||
|
|
|
|||
|
|
const findings = []; // 异常发现
|
|||
|
|
let reportSent = false;
|
|||
|
|
|
|||
|
|
// ---------- 1. 进程检查与强杀 ----------
|
|||
|
|
let psOut = '';
|
|||
|
|
try { psOut = execSync("ps aux | grep -E 'daily-runner|zhaopin-scrape' | grep -v grep || true", { encoding: 'utf8' }); } catch { }
|
|||
|
|
|
|||
|
|
const lines = psOut.split('\n').filter(l => l.trim());
|
|||
|
|
if (lines.length === 0) {
|
|||
|
|
log('无残留进程 ✅');
|
|||
|
|
} else {
|
|||
|
|
for (const line of lines) {
|
|||
|
|
const pid = line.trim().split(/\s+/)[1];
|
|||
|
|
// 卡死检测：cputime vs etime
|
|||
|
|
let hung = false, detail = '';
|
|||
|
|
try {
|
|||
|
|
const pi = execSync(`ps -p ${pid} -o pid,stat,etime,cputime,rss,cmd --no-headers`, { encoding: 'utf8' }).trim();
|
|||
|
|
detail = pi;
|
|||
|
|
const parts = pi.split(/\s+/);
|
|||
|
|
const etime = parts[2] || '', cputime = parts[3] || '';
|
|||
|
|
const toSec = (t) => { // etime: [[dd-]hh:]mm:ss
|
|||
|
|
let s = 0;
|
|||
|
|
const m = t.match(/(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+)/);
|
|||
|
|
if (m) s = (+m[1] || 0) * 86400 + (+m[2] || 0) * 3600 + (+m[3]) * 60 + (+m[4]);
|
|||
|
|
return s;
|
|||
|
|
};
|
|||
|
|
const cSec = toSec(cputime), eSec = toSec(etime);
|
|||
|
|
if (cSec === 0 || (eSec > 600 && cSec / eSec < 0.01)) hung = true;
|
|||
|
|
} catch { }
|
|||
|
|
findings.push(`18点后仍在运行${hung ? '（疑似卡死）' : ''}: ${line.trim().slice(0, 140)}`);
|
|||
|
|
try { process.kill(parseInt(pid, 10), 'SIGKILL'); log(`已强杀 PID ${pid}`); }
|
|||
|
|
catch (e) { log(`杀 ${pid} 失败: ${e.message}`); findings.push(`强杀失败 PID ${pid}: ${e.message}`); }
|
|||
|
|
}
|
|||
|
|
// 复查
|
|||
|
|
try {
|
|||
|
|
const left = execSync("ps aux | grep -E 'daily-runner|zhaopin-scrape' | grep -v grep || true", { encoding: 'utf8' }).trim();
|
|||
|
|
if (left) findings.push(`强杀后仍有残留：\n${left.slice(0, 300)}`);
|
|||
|
|
} catch { }
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ---------- 2. 状态文件检查 ----------
|
|||
|
|
for (const acct of ['hr1', 'hr2']) {
|
|||
|
|
const f = path.join(STATE_DIR, `state-${acct}.json`);
|
|||
|
|
try {
|
|||
|
|
const s = JSON.parse(fs.readFileSync(f, 'utf8'));
|
|||
|
|
if (s.date !== today()) findings.push(`${acct}: 今日无状态记录（runner 未启动或启动即失败）`);
|
|||
|
|
else log(`${acct}: 今日状态=${s.status} endedAt=${s.endedAt}`);
|
|||
|
|
} catch {
|
|||
|
|
findings.push(`${acct}: 状态文件缺失（runner 未启动）`);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ---------- 3. 日报补发 ----------
|
|||
|
|
const sentFlag = path.join(STATE_DIR, `report-sent-${today()}`);
|
|||
|
|
if (fs.existsSync(sentFlag)) {
|
|||
|
|
reportSent = true;
|
|||
|
|
log('日报今日已由 runner 发送 ✅');
|
|||
|
|
} else {
|
|||
|
|
log('日报未发送，补发中...');
|
|||
|
|
try {
|
|||
|
|
const env = { ...process.env };
|
|||
|
|
// 从状态文件取聊天剩余
|
|||
|
|
for (const [acct, key] of [['hr1', 'CHAT_LEFT_HR1'], ['hr2', 'CHAT_LEFT_HR2']]) {
|
|||
|
|
try { const s = JSON.parse(fs.readFileSync(path.join(STATE_DIR, `state-${acct}.json`), 'utf8')); if (s.chatLeft != null) env[key] = String(s.chatLeft); } catch { }
|
|||
|
|
}
|
|||
|
|
const md = execSync(`node ${SCRIPTS}/daily-report.js`, { cwd: SCRIPTS, encoding: 'utf8', timeout: 60000, env });
|
|||
|
|
fs.writeFileSync(`/tmp/daily-report-enforce-${today()}.md`, md);
|
|||
|
|
execSync(`${HERMES} send -t "feishu:${GROUP}" -f /tmp/daily-report-enforce-${today()}.md`, { stdio: ['pipe', 'pipe', 'pipe'], timeout: 60000 });
|
|||
|
|
fs.writeFileSync(sentFlag, new Date().toISOString());
|
|||
|
|
reportSent = true;
|
|||
|
|
log('日报已补发到群 ✅');
|
|||
|
|
} catch (e) { findings.push(`日报补发失败: ${String(e.message).slice(0, 150)}`); }
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ---------- 4. 汇总输出 ----------
|
|||
|
|
console.log('\n========== 18点收队结果 ==========');
|
|||
|
|
console.log(`日报: ${reportSent ? '✅ 已发送' : '❌ 未发送'}`);
|
|||
|
|
console.log(`异常: ${findings.length === 0 ? '无 ✅' : findings.length + ' 项 ⚠️'}`);
|
|||
|
|
for (const f of findings) console.log(` ⚠️ ${f}`);
|
|||
|
|
console.log('==================================');
|
|||
|
|
process.exit(0);
|