mzl92
109 lines
4.9 KiB
JavaScript
109 lines
4.9 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* 18:00 强制收队 + 异常检测 v1.0（2026-08-26）
|
||
* 由 cron 在北京时间 18:05 触发（UTC 10:05）。
|
||
*
|
||
* 职责：
|
||
* 1. 强杀所有仍在运行的 daily-runner / zhaopin-scrape 进程（正常应已自行结束）
|
||
* 2. 异常检测：
|
||
* a. 18点仍在跑的进程 → 判定"未按时收队"（含失联/卡死）
|
||
* b. 卡死检测：进程 cputime 为 0 或远小于 etime（pitfalls #81）
|
||
* c. 状态文件检查：state-hrX.json 缺失/非当日 → 该账号今日未正常运行
|
||
* 3. 若日报今日未发（runner 异常没发成）→ 补发生成日报到群
|
||
* 4. 输出检测结果（cron agent 转发给用户私聊）
|
||
*/
|
||
const { execSync } = require('child_process');
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
|
||
const SCRIPTS = '/opt/data/skills/recruitment-resume-screener/scripts';
|
||
const STATE_DIR = '/opt/data/logs/recruitment/state';
|
||
fs.mkdirSync(STATE_DIR, { recursive: true });
|
||
const GROUP = 'oc_d09c77b62ca5f90f6699830fbc695c44'; // 2026-08-26 修正:只发私聊
|
||
const HERMES = '/opt/hermes/.venv/bin/hermes';
|
||
const bjNow = () => new Date(Date.now() + 8 * 3600 * 1000);
|
||
const today = () => bjNow().toISOString().slice(0, 10);
|
||
const log = (m) => console.log(`[enforce] ${m}`);
|
||
|
||
const findings = []; // 异常发现
|
||
let reportSent = false;
|
||
|
||
// ---------- 1. 进程检查与强杀 ----------
|
||
let psOut = '';
|
||
try { psOut = execSync("ps aux | grep -E 'daily-runner|zhaopin-scrape' | grep -v grep || true", { encoding: 'utf8' }); } catch { }
|
||
|
||
const lines = psOut.split('\n').filter(l => l.trim());
|
||
if (lines.length === 0) {
|
||
log('无残留进程 ✅');
|
||
} else {
|
||
for (const line of lines) {
|
||
const pid = line.trim().split(/\s+/)[1];
|
||
// 卡死检测：cputime vs etime
|
||
let hung = false, detail = '';
|
||
try {
|
||
const pi = execSync(`ps -p ${pid} -o pid,stat,etime,cputime,rss,cmd --no-headers`, { encoding: 'utf8' }).trim();
|
||
detail = pi;
|
||
const parts = pi.split(/\s+/);
|
||
const etime = parts[2] || '', cputime = parts[3] || '';
|
||
const toSec = (t) => { // etime: [[dd-]hh:]mm:ss
|
||
let s = 0;
|
||
const m = t.match(/(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+)/);
|
||
if (m) s = (+m[1] || 0) * 86400 + (+m[2] || 0) * 3600 + (+m[3]) * 60 + (+m[4]);
|
||
return s;
|
||
};
|
||
const cSec = toSec(cputime), eSec = toSec(etime);
|
||
if (cSec === 0 || (eSec > 600 && cSec / eSec < 0.01)) hung = true;
|
||
} catch { }
|
||
findings.push(`18点后仍在运行${hung ? '（疑似卡死）' : ''}: ${line.trim().slice(0, 140)}`);
|
||
try { process.kill(parseInt(pid, 10), 'SIGKILL'); log(`已强杀 PID ${pid}`); }
|
||
catch (e) { log(`杀 ${pid} 失败: ${e.message}`); findings.push(`强杀失败 PID ${pid}: ${e.message}`); }
|
||
}
|
||
// 复查
|
||
try {
|
||
const left = execSync("ps aux | grep -E 'daily-runner|zhaopin-scrape' | grep -v grep || true", { encoding: 'utf8' }).trim();
|
||
if (left) findings.push(`强杀后仍有残留：\n${left.slice(0, 300)}`);
|
||
} catch { }
|
||
}
|
||
|
||
// ---------- 2. 状态文件检查 ----------
|
||
for (const acct of ['hr1', 'hr2']) {
|
||
const f = path.join(STATE_DIR, `state-${acct}.json`);
|
||
try {
|
||
const s = JSON.parse(fs.readFileSync(f, 'utf8'));
|
||
if (s.date !== today()) findings.push(`${acct}: 今日无状态记录（runner 未启动或启动即失败）`);
|
||
else log(`${acct}: 今日状态=${s.status} endedAt=${s.endedAt}`);
|
||
} catch {
|
||
findings.push(`${acct}: 状态文件缺失（runner 未启动）`);
|
||
}
|
||
}
|
||
|
||
// ---------- 3. 日报补发 ----------
|
||
const sentFlag = path.join(STATE_DIR, `report-sent-${today()}`);
|
||
if (fs.existsSync(sentFlag)) {
|
||
reportSent = true;
|
||
log('日报今日已由 runner 发送 ✅');
|
||
} else {
|
||
log('日报未发送，补发中...');
|
||
try {
|
||
const env = { ...process.env };
|
||
// 从状态文件取聊天剩余
|
||
for (const [acct, key] of [['hr1', 'CHAT_LEFT_HR1'], ['hr2', 'CHAT_LEFT_HR2']]) {
|
||
try { const s = JSON.parse(fs.readFileSync(path.join(STATE_DIR, `state-${acct}.json`), 'utf8')); if (s.chatLeft != null) env[key] = String(s.chatLeft); } catch { }
|
||
}
|
||
const md = execSync(`node ${SCRIPTS}/daily-report.js`, { cwd: SCRIPTS, encoding: 'utf8', timeout: 60000, env });
|
||
fs.writeFileSync(`/tmp/daily-report-enforce-${today()}.md`, md);
|
||
execSync(`${HERMES} send -t "feishu:${GROUP}" -f /tmp/daily-report-enforce-${today()}.md`, { stdio: ['pipe', 'pipe', 'pipe'], timeout: 60000 });
|
||
fs.writeFileSync(sentFlag, new Date().toISOString());
|
||
reportSent = true;
|
||
log('日报已补发到群 ✅');
|
||
} catch (e) { findings.push(`日报补发失败: ${String(e.message).slice(0, 150)}`); }
|
||
}
|
||
|
||
// ---------- 4. 汇总输出 ----------
|
||
console.log('\n========== 18点收队结果 ==========');
|
||
console.log(`日报: ${reportSent ? '✅ 已发送' : '❌ 未发送'}`);
|
||
console.log(`异常: ${findings.length === 0 ? '无 ✅' : findings.length + ' 项 ⚠️'}`);
|
||
for (const f of findings) console.log(` ⚠️ ${f}`);
|
||
console.log('==================================');
|
||
process.exit(0);
|