xinyouhe/fdesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature/mzl_dev'

Browse Source
This commit is contained in:
mzl
2025-08-04 14:03:57 +08:00
parent 8f74999b47
commit b2d4119e46
25 changed files with 2066 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/command/DomainCommand.php
+3
View File
@@ -11,6 +11,9 @@ use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use support\Db;
/**
* 主要处理询单详情中的图片地址
*/
class DomainCommand extends Command
{
protected static $defaultName = 'domain';
app/command/EmailDetailCommand.php
+206
View File
@@ -0,0 +1,206 @@
<?php
namespace app\command;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Helper\Table;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use support\Db;
/**
* 拉取邮箱的详情入库（根据emai_uid）
*/
class EmailDetailCommand extends Command
{
protected static $defaultName = 'email-detail';
protected static $defaultDescription = '拉取 detail 入库';
protected function configure()
{
$this->addOption(
'per-page', 'p',
InputOption::VALUE_OPTIONAL,
'每批处理多少条详情',
config('imap.per_page', 50)
);
}
protected function execute(InputInterface $input, OutputInterface $output): int
{
try {
$cfg = config('imap');
$mailbox = $cfg['mailbox'];
$username = $cfg['username'];
$password = $cfg['password'];
$batch = (int)$input->getOption('per-page');
// 打开 IMAP
$inbox = @imap_open($mailbox, $username, $password);
if (!$inbox) {
$output->writeln("<error>IMAP 连接失败：". imap_last_error() ."</error>");
return Command::FAILURE;
}
// 阶段二：逐批拉取详情并写入 emails 表
$output->writeln("依次拉取详情并写入 emails 表 …");
while (true) {
$pending = Db::table('email_uids')
->where('status', 0)
// ->where('uid', '>', 13399)
->orderBy('uid', 'desc')
->limit(2)
->get();
if (count($pending) === 0) {
$output->writeln("<info>所有 UID 均已处理完毕。</info>");
break;
}
$output->writeln("处理下一批 " . count($pending) . " 条 …");
foreach ($pending as $row) {
$uid = $row->uid;
// 包裹整行处理在 try/catch 中，任何异常都不会中断循环
try {
$output->writeln("---- 开始处理 UID={$uid} ----");
// 1. fetch overview
$ov = @imap_fetch_overview($inbox, $uid, FT_UID)[0] ?? null;
if (!$ov) {
$err = imap_last_error() ?: '未知错误';
$output->writeln("<error>UID={$uid} overview 失败：{$err}</error>");
$this->logError("UID={$uid} overview 失败：{$err}");
continue;
}
$subject = mb_convert_encoding($ov->subject ?? '', 'UTF-8', 'GBK,UTF-8');
$from = $ov->from ?? '';
$date = $ov->date ?? '';
$output->writeln("UID={$uid} overview 成功，subject={$subject}");
// 2. fetch body
$body = $this->getMailBody($inbox, $uid, true);
$output->writeln("UID={$uid} body 长度：" . strlen($body));
// 3. sanitize & insert
$body = $this->sanitizeUtf8($body);
$now = time();
try {
Db::table('emails')->insert([
'uid' => $uid,
'subject' => $subject,
'from' => $from,
'date' => $date,
'body' => $body,
'created_at' => $now,
'updated_at' => $now,
]);
$output->writeln("<info>UID={$uid} 写入 emails 成功</info>");
} catch (\Throwable $e) {
$output->writeln("<error>UID={$uid} 写入失败：{$e->getMessage()}</error>");
$this->logError("UID={$uid} 写入失败：{$e->getMessage()}");
continue;
}
// 4. 标记已处理
try {
Db::table('email_uids')
->where('uid', $uid)
->update([
'status' => 1,
'processed_at' => $now,
]);
$output->writeln("<info>UID={$uid} 标记为已处理</info>");
} catch (\Throwable $e) {
$output->writeln("<error>UID={$uid} 标记状态失败：{$e->getMessage()}</error>");
$this->logError("UID={$uid} 标记状态失败：{$e->getMessage()}");
// 即使更新 status 失败，也继续下一条
}
} catch (\Throwable $e) {
// 捕获任何非预期错误
$output->writeln("<error>UID={$uid} 处理异常：{$e->getMessage()}</error>");
$this->logError("UID={$uid} 处理异常：{$e->getMessage()}");
continue;
}
}
// 批次结束后稍作休眠，防止服务器过载
sleep(1);
}
imap_close($inbox);
$output->writeln("\n<info>✅ 全部邮件处理完成。</info>");
return Command::SUCCESS;
} catch (\Throwable $th) {
dpr($th);
return Command::FAILURE;
}
}
/**
* 将任意字符串清洗为有效 UTF-8，移除或替换无效字节
*/
protected function sanitizeUtf8(string $text): string
{
return iconv('UTF-8', 'UTF-8//IGNORE', $text);
}
/**
* 记录错误到日志
*/
protected function logError(string $msg): void
{
$line = '[' . date('Y-m-d H:i:s') . '] ' . $msg . "\n";
file_put_contents(
runtime_path('logs/email_errors.log'),
$line,
FILE_APPEND | LOCK_EX
);
}
/**
* 按编码解码并保留 "\r\n"
*/
protected function getMailBody($inbox, int $id, bool $isUid = false): string
{
$flags = $isUid ? FT_UID : 0;
$structure = imap_fetchstructure($inbox, $id, $flags);
if (!empty($structure->parts)) {
// HTML 部分
foreach ($structure->parts as $i => $part) {
if (isset($part->subtype) && strtolower($part->subtype) === 'html') {
$data = imap_fetchbody($inbox, $id, (string)($i+1), $flags);
return $this->decodeBody($data, $part->encoding);
}
}
// 文本部分
foreach ($structure->parts as $i => $part) {
if (isset($part->subtype) && strtolower($part->subtype) === 'plain') {
$data = imap_fetchbody($inbox, $id, (string)($i+1), $flags);
return $this->decodeBody($data, $part->encoding);
}
}
// 原始 body
return imap_body($inbox, $id, $flags);
}
return imap_body($inbox, $id, $flags);
}
/**
* 按编码解码
*/
protected function decodeBody(string $body, int $encoding): string
{
switch ($encoding) {
case 3: return imap_base64($body);
case 4: return imap_qprint($body);
default: return $body;
}
}
}
app/command/EmailParseCommand.php
+144
View File
@@ -0,0 +1,144 @@
<?php
namespace app\command;
use support\Db;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
/**
* 将拉取邮箱的详情进行清洗，找出询盘的邮件入库（根据emai_detail）
*/
class EmailParseCommand extends Command
{
protected static $defaultName = 'email-parse';
protected static $defaultDescription = '邮件清洗';
protected function configure()
{
// 不再接收参数，固定每次 100 条
}
protected function execute(InputInterface $input, OutputInterface $output): int
{
$batchSize = 100; // 每次处理 100 条
$page = 0;
// 明确定义 parsed_contacts 表里要插入的所有列（顺序无所谓，但要全部包含）
$fields = [
'email_id',
'date',
'email',
'name',
'tel',
'company',
'country',
'form_url',
'message',
'ip',
'iplocation',
'ilang',
'mailto',
'navigator_language',
'navigator_platform',
'screen_width',
'screen_height',
'ucountry',
];
while (true) {
$rows = Db::table('emails')
->orderBy('id', 'asc')
->limit($batchSize)
->offset($page * $batchSize)
->get();
if ($rows->isEmpty()) {
break;
}
$toInsert = [];
foreach ($rows as $row) {
$parsed = $this->parseHtml($row->body);
if ($parsed === null) {
$output->writeln(" ↳ [{$row->id}] skip: not a contact-form HTML");
continue;
}
// 严格按 $fields 顺序填充每条记录，缺失的字段补 null
$record = [];
foreach ($fields as $col) {
if ($col === 'email_id') {
$record[$col] = $row->id;
} else {
$record[$col] = $parsed[$col] ?? null;
}
}
$toInsert[] = $record;
}
if (!empty($toInsert)) {
Db::table('email_parsed_result')->insert($toInsert);
$output->writeln("Inserted batch #" . ($page + 1) . ", count: " . count($toInsert));
}
$page++;
}
$output->writeln("✔ All done.");
return Command::SUCCESS;
}
/**
* 解析 HTML，找不到 <h2>Preview</h2> 则返回 null，
* 否则提取所有 <span>Key:</span> Value 为小写蛇形 key=>value 数组
*/
protected function parseHtml(string $html): ?array
{
$doc = new \DOMDocument();
libxml_use_internal_errors(true);
$doc->loadHTML('<?xml encoding="utf-8"?>' . $html);
libxml_clear_errors();
$xpath = new \DOMXPath($doc);
// 类型检查
if ($xpath->query("//h2[text()='Preview']")->length === 0) {
return null;
}
$nodes = $xpath->query("//div[span]");
$data = [];
/** @var \DOMElement $node */
foreach ($nodes as $node) {
$span = $xpath->query("span", $node)->item(0);
if (! $span) {
continue;
}
// 原始键名
$key = trim(str_replace(':', '', $span->textContent));
// 转成小写蛇形
$col = strtolower(str_replace(' ', '_', $key));
// 获取值
$a = $xpath->query(".//a", $node)->item(0);
if ($a && $a->hasAttribute('href')) {
$value = trim($a->getAttribute('href'));
} else {
$clone = $node->cloneNode(true);
$spanClone = $xpath->query(".//span", $clone)->item(0);
if ($spanClone) {
$spanClone->parentNode->removeChild($spanClone);
}
$value = trim($clone->textContent);
}
$data[$col] = $value;
}
return $data;
}
}
app/command/EmailUidCommand.php
+185
View File
@@ -0,0 +1,185 @@
<?php
namespace app\command;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Helper\Table;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use support\Db;
/**
* 拉取邮箱的UID入库
*/
class EmailUidCommand extends Command
{
protected static $defaultName = 'email-uid';
protected static $defaultDescription = '拉取 UID 入库';
// protected function configure()
// {
// $this->addOption(
// 'per-page', 'p',
// InputOption::VALUE_OPTIONAL,
// '每批处理多少条详情',
// config('imap.per_page', 50)
// );
// }
protected function execute(InputInterface $input, OutputInterface $output): int
{
try {
$cfg = config('imap');
$mailbox = $cfg['mailbox'];
$username = $cfg['username'];
$password = $cfg['password'];
// $batch = (int)$input->getOption('per-page');
$retryTimes = 3;
$retryDelay = 10; // 秒
$inbox = false;
$attempt = 0;
$maxUid = Db::table('email_uids')->max('uid');
while ($attempt < $retryTimes) {
$attempt++;
$output->writeln("尝试第 {$attempt} 次连接 IMAP...");
$inbox = @imap_open($mailbox, $username, $password);
if ($inbox) {
break;
}
$error = imap_last_error();
$output->writeln("<comment>第 {$attempt} 次失败：{$error}</comment>");
// 若出现 too many login failures 立即暂停一段时间
if (stripos($error, 'Too many login failures') !== false) {
$output->writeln("<error>检测到登录失败次数过多，暂停 60 秒避免封禁</error>");
sleep(60);
} else {
sleep($retryDelay);
}
}
if (!$inbox) {
$output->writeln("<error>最终仍然连接失败：" . imap_last_error() . "</error>");
return Command::FAILURE;
}
// 拉取 UID 并入库
$output->writeln("开始拉取 UID 并入库 …");
$uids = imap_search($inbox, 'ALL', SE_UID) ?: [];
$uids = array_filter($uids, function($uid) use ($maxUid) {
return $uid > $maxUid;
});
sort($uids);
$now = time();
$count = 0;
foreach ($uids as $uid) {
try {
Db::table('email_uids')->insert([
'uid' => $uid,
'fetched_at' => $now,
'email' => $cfg['username']
]);
$count++;
$output->writeln("UID：".$uid." 已写入数据库");
} catch (\Throwable $e) {
$output->writeln("<comment>{$e->getMessage()}</comment>");
}
}
imap_close($inbox);
$output->writeln(date('Y-m-d H:i:s')."：共拉到 {$count} 个 UID 并写入 email_uids 表。");
return Command::SUCCESS;
} catch (\Throwable $th) {
dpr($th);
return Command::FAILURE;
}
}
}
/**
* 获取指定格式的日期字符串
*
* @param string|null $input_date 可选，指定的日期（可以是任何PHP可解析的日期格式）
* @return string 返回格式为 'd-M-Y' 的日期字符串（如 '28-Jul-2025'）
*/
function getImapFormattedDate($input_date = null) {
// 如果有传入日期参数，尝试解析
if ($input_date !== null) {
$timestamp = strtotime($input_date);
if ($timestamp === false) {
// 如果日期解析失败，使用当前日期并记录警告
trigger_error("无法解析提供的日期: '$input_date'，将使用当前日期", E_USER_WARNING);
$timestamp = time();
}
} else {
// 如果没有提供日期，使用当前日期
$timestamp = time();
}
// 格式化为 IMAP 需要的日期格式 (如 '28-Jul-2025')
return date('d-M-Y', $timestamp);
}
// // 使用示例
// echo getImapFormattedDate(); // 输出当天的日期，如 '31-Jul-2025'
// echo getImapFormattedDate('2025-07-28'); // 输出 '28-Jul-2025'
// echo getImapFormattedDate('28/07/2025'); // 输出 '28-Jul-2025'
// echo getImapFormattedDate('next Monday'); // 输出下个周一的日期
/*
// imap_search() 条件
// 1. 基本日期搜索（按接收日期）
// 搜索今天收到的邮件
$today = date("d-M-Y");
$uids = imap_search($inbox, 'SINCE "' . $today . '"', SE_UID);
// 搜索特定日期之后的邮件
$since_date = '15-Jul-2023';
$uids = imap_search($inbox, 'SINCE "' . $since_date . '"', SE_UID);
// 搜索特定日期之前的邮件
$before_date = '20-Jul-2023';
$uids = imap_search($inbox, 'BEFORE "' . $before_date . '"', SE_UID);
// 搜索特定日期的邮件
$on_date = '18-Jul-2023';
$uids = imap_search($inbox, 'ON "' . $on_date . '"', SE_UID);
// 2. 日期范围搜索
// 搜索2023年7月15日到2023年7月20日之间的邮件
$start_date = '15-Jul-2023';
$end_date = '20-Jul-2023';
$uids = imap_search($inbox, 'SINCE "' . $start_date . '" BEFORE "' . $end_date . '"', SE_UID);
// 3. 结合日期和其他条件
// 搜索2023年7月以后未读的邮件
$date = '1-Jul-2023';
$uids = imap_search($inbox, 'SINCE "' . $date . '" UNSEEN', SE_UID);
// 搜索2023年7月来自某人的邮件
$date = '1-Jul-2023';
$from = 'example@domain.com';
$uids = imap_search($inbox, 'SINCE "' . $date . '" FROM "' . $from . '"', SE_UID);
// 4. 日期格式注意事项
// - IMAP日期格式通常为 d-M-Y，例如 15-Jul-2023
// - 月份使用英文缩写（Jan, Feb, Mar, Apr, May, Jun, Jul, Aug, Sep, Oct, Nov, Dec）
// - 日期格式必须用双引号括起来
所有ID
imap_search($inbox, 'ALL', SE_UID)
*/
app/command/FixInquiryLogOrderNo.php
+3
View File
@@ -7,6 +7,9 @@ use Symfony\Component\Console\Output\OutputInterface;
use support\Db;
use support\Log;
/**
* 处理询单中的编号重复
*/
class FixInquiryLogOrderNo extends Command
{
protected static $defaultName = 'sn';
app/command/TbCommand.php
+13 -5
View File
@@ -9,7 +9,11 @@ use Symfony\Component\Console\Helper\Table;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use support\Db;
use app\common\utils\email\Emails;
/**
* 批量处理用户编号（也用于调试）
*/
class TbCommand extends Command
{
protected static $defaultName = 'tb-db';
@@ -17,12 +21,16 @@ class TbCommand extends Command
protected function execute(InputInterface $input, OutputInterface $output): int
{
$user = new User();
$list = $user->all();
// $user = new User();
// $list = $user->all();
foreach ($list as $item) {
// UserSn::where('user_sn', $item->sn)->update(['user_id' => $item->id]);
}
// foreach ($list as $item) {
// // UserSn::where('user_sn', $item->sn)->update(['user_id' => $item->id]);
// }
$m = new Emails();
$m->process();
return self::SUCCESS;
}
}