xinyouhe/fdesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

二手车信息爬虫服务

Browse Source
This commit is contained in:
mzl
2025-11-24 18:29:11 +08:00
parent 1d9fbb9032
commit 79d19a53e6
3 changed files with 664 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/common/service/UsedCarCrawlService.php
+647
View File
@@ -0,0 +1,647 @@
<?php
namespace app\common\service;
use GuzzleHttp\Client;
use QL\QueryList;
use support\Redis;
use support\Db;
use support\exception\BusinessException;
class UsedCarCrawlService
{
/**
* 插入队列url
*
* $scene 二手信息 usedcar 配置信息 config
*/
public function iUrl(string $url = '', string $scene = 'usedcar', int $id = 0)
{
return Redis::lpush('usedcar:urls', json_encode([
'scene' => $scene,
'url' => $url,
'id' => $id
], JSON_UNESCAPED_UNICODE));
}
/**
* 获取队列url
*
* $scene 二手信息 usedcar 配置信息 config
*/
public function rUrl()
{
return json_decode(Redis::rPop('usedcar:urls'), true);
}
/**
* 执行抓取
*/
public function doData()
{
try {
$raw = $this->rUrl();
if (!$raw) {
return false;
}
echo '开始抓取~：'.date('Y-m-d H:i:s').PHP_EOL;
$html = $this->getHtml($raw['url']);
switch ($raw['scene']) {
case 'usedcar':
$info = $this->getUsedDetail($html, $raw['url']);
$id = $this->saveUsedcarBasicToDb($info);
$this->iUrl($info['car_id_url'], 'config', $id);
echo '一阶段 抓取成功~：'.date('Y-m-d H:i:s').PHP_EOL;
break;
case 'config':
$data = $this->getConfigDetail($html);
$this->saveUsedcarConfigToDb($data, $raw['id']);
echo '二阶段 抓取成功~：'.date('Y-m-d H:i:s').PHP_EOL;
break;
}
return true;
} catch (\Throwable $th) {
throw new BusinessException($th->getMessage().'|'.$th->getLine().'|'.$th->getFile());
}
}
/**
* 获取html
*/
public function getHtml($url)
{
$client = new Client();
$res = $client->request('GET', $url, [
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
. 'AppleWebKit/537.36 (KHTML, like Gecko) '
. 'Chrome/120.0.0.0 Safari/537.36',
'Accept-Language' => 'zh-CN,zh;q=0.9',
'Referer' => 'https://www.dongchedi.com/'
]
]);
return (string)$res->getBody();
}
/**
* 获取二手信息
*/
public function getUsedDetail(string $html = '', string $url = '')
{
$ql = QueryList::html($html);
$title = $ql->find('h1')->text();
$price = $ql->find('.head-info_price-wrap__Y4bxi > p')->text();
$pageText = $ql->find('body')->text();
$jsonText = $ql->find('#__NEXT_DATA__')->text();
// 新车指导价
$newPrice = '';
if ($pageText) {
if (preg_match('/新车指导价[:：]\s*([^\r\n万]*万)/u', $pageText, $m)) {
$newPrice = trim(preg_replace('/[^0-9.]/', '', $m[1]));
}
}
// 比新车省
$priceDifference = '';
if ($pageText) {
if (preg_match('/比新车省[:：]\s*([^\r\n万]*万)/u', $pageText, $m)) {
$priceDifference = trim(preg_replace('/[^0-9.]/', '', $m[1]));
}
}
// 车况介绍
$conditionDesc = '';
if ($pageText) {
if (preg_match('/车况介绍\s*(.*?)\s*推荐理由/su', $pageText, $m)) {
$conditionDesc = trim($m[1]);
}
}
$jsonArray = json_decode($jsonText, true);
// 车辆相册
$images = $jsonArray['props']['pageProps']['skuDetail']['head_images'] ?? [];
// 车辆ID
$carId = $jsonArray['props']['pageProps']['skuDetail']['car_info']['car_id'] ?? 0;
// 车辆年份
$year = $jsonArray['props']['pageProps']['skuDetail']['car_info']['year'] ?? 0;
// 车辆ID URL
$carIdUrl = $carId ? ('https://www.dongchedi.com/auto/params-carIds-' . $carId) : '';
// 能源类型
$fuelForm = $jsonArray['props']['pageProps']['skuDetail']['car_config_overview']['power']['fuel_form'] ?? '';
// 使用里程
$importantText = $jsonArray['props']['pageProps']['skuDetail']['important_text'] ?? '';
$importantTextArray = explode(' | ', $importantText);
$usageMileage = str_replace('万公里', '', isset($importantTextArray[1]) ? $importantTextArray[1] : '');
// 车辆名称 品牌、车系、车型
$brandName = $jsonArray['props']['pageProps']['skuDetail']['car_info']['brand_name'] ?? '';
$seriesName = $jsonArray['props']['pageProps']['skuDetail']['car_info']['series_name'] ?? '';
$carName = $jsonArray['props']['pageProps']['skuDetail']['car_info']['car_name'] ?? '';
// 商家信息
$shopInfo = $jsonArray['props']['pageProps']['skuDetail']['shop_info'] ?? [];
$formatShopInfo = [
'shop_id' => $shopInfo['shop_id'] ?? 0,
'shop_name' => $shopInfo['shop_short_name'] ?? '',
'shop_address' => $shopInfo['shop_address'] ?? ''
];
// 简介参数
$otherParams = $jsonArray['props']['pageProps']['skuDetail']['other_params'] ?? [];
// 排量
$displacement = $this->getOtherParamValue($otherParams, '排量');
// 变速箱
$gearbox = $this->getOtherParamValue($otherParams, '变速箱');
// 格式化车辆信息
$formatCarInfo = [
'displacement' => $displacement,
'gearbox' => $gearbox
];
// 整理车辆信息
$info['used_car_id'] = $this->getLastIdFromUrl($url);
$info['car_id'] = $carId;
$info['url'] = $url;
$info['title'] = $title;
$info['price'] = str_replace('万', '', $this->decodeDongchediPrice($price));
$info['new_price'] = str_replace('万', '', $newPrice);
$info['price_difference'] = str_replace('万', '', $priceDifference);
$info['condition_desc'] = $conditionDesc;
$info['images'] = json_encode($images);
$info['car_id_url'] = $carIdUrl;
$info['used_detail'] = $jsonText;
$info['format_shop_info'] = json_encode($formatShopInfo);
$info['fuel_form'] = $fuelForm;
$info['classify_energy'] = $this->classifyEnergy($fuelForm);
$info['usage_mileage'] = $usageMileage;
$info['brand_name'] = $brandName;
$info['series_name'] = $seriesName;
$info['car_name'] = $carName;
$info['year'] = $year;
$info['format_car_info'] = json_encode($formatCarInfo);
return $info;
}
/**
* 获取配置信息
*/
public function getConfigDetail(string $html = '')
{
$ql = QueryList::html($html);
$jsonText = $ql->find('#__NEXT_DATA__')->text();
$raw = json_decode($jsonText, true);
$properties = $raw['props']['pageProps']['rawData']['properties'] ?? [];
$carInfo = $raw['props']['pageProps']['rawData']['car_info'][0] ?? [];
$valuesMap = $this->buildValuesMapFromCarInfo($carInfo);
$groups = $this->buildCarSpecGroups($properties, $valuesMap);
$data['format_car_info'] = [
'market_time' => $this->getFieldValue($groups, 'market_time'), // 上市时间
'driver_form' => $this->getFieldValue($groups, 'driver_form'), // 驱动形式
'seat_count' => $this->getFieldValue($groups, 'seat_count'), // 座位数
'engine_description' => $this->getFieldValue($groups, 'engine_description'), // 发动机
'engine_max_power_horsepower' => $this->getFieldValue($groups, 'engine_max_power_horsepower'), // 发动机最大功率
'electric_description' => $this->getFieldValue($groups, 'electric_description'), //电动机
'battery_capacity' => $this->getFieldValue($groups, 'battery_capacity'), //电池容量
'battery_type' => $this->getFieldValue($groups, 'battery_type'), // 电池类型
'recharge_mileage' => $this->getFieldValue($groups, 'recharge_mileage') // 续航里程
];
$data['groups'] = $groups;
return $data;
}
public function getFuelFormByConfigInfo(array $data)
{
$fuelForm = null;
foreach ($data[0]['items'] as $item) {
if ($item['key'] === 'fuel_form') {
$fuelForm = $item['value'];
break;
}
}
return $fuelForm;
}
/**
* 从车辆配置 JSON 数据中根据字段 key 获取对应的 value
*
* @param string|array $data JSON 字符串 或 已解析的数组
* @param string $field 要查找的字段 key（如 'fuel_form'）
* @return string|null 找到则返回 value，否则返回 null
*/
public function getFieldValue($data, string $field)
{
// 如果传入的是 JSON 字符串，先解码
if (is_string($data)) {
$data = json_decode($data, true);
if (json_last_error() !== JSON_ERROR_NONE) {
return null; // JSON 格式错误
}
}
// 遍历第一层（假设结构是 [ { "items": [...] } ]）
foreach ($data as $group) {
if (isset($group['items']) && is_array($group['items'])) {
foreach ($group['items'] as $item) {
if (isset($item['key']) && $item['key'] === $field) {
return $item['value'] ?? null;
}
}
}
}
return null; // 未找到
}
/**
* 从 other_params 数组中根据 name 获取对应的 value
*
* @param array $otherParams 格式为 [ ['name' => 'xxx', 'value' => 'yyy'], ... ]
* @param string $name 要查找的字段名称（如 '上牌地'）
* @return string|null 找到则返回 value，否则返回 null
*/
public function getOtherParamValue(array $otherParams, string $name): ?string
{
foreach ($otherParams as $item) {
if (isset($item['name']) && $item['name'] === $name) {
return $item['value'] ?? null;
}
}
return null;
}
/**
* 获取能源类型
*/
public function classifyEnergy($input)
{
$ce = ['汽油', '柴油', '天然气'];
$re = ['纯电', '电动', '氢能', '氢燃料', '太阳能', '油电混动', '混动', '插电混动', '插电式', '增程式', '增程'];
// 标准化输入：去除空格等
$input = trim($input);
// 1. 精确或子串匹配（优先）
foreach ($re as $keyword) {
if (strpos($input, $keyword) !== false) {
return 'RE'; // 新能源
}
}
foreach ($ce as $keyword) {
if (strpos($input, $keyword) !== false) {
return 'CE'; // 传统能源
}
}
// 2. 如果没匹配上，尝试模糊匹配（相似度 > 80%）
$bestMatch = null;
$bestScore = 0;
$bestType = null;
// 检查 RE
foreach ($re as $keyword) {
similar_text($input, $keyword, $percent);
if ($percent > $bestScore && $percent >= 80) {
$bestScore = $percent;
$bestMatch = $keyword;
$bestType = 'RE';
}
}
// 检查 CE
foreach ($ce as $keyword) {
similar_text($input, $keyword, $percent);
if ($percent > $bestScore && $percent >= 80) {
$bestScore = $percent;
$bestMatch = $keyword;
$bestType = 'CE';
}
}
return $bestType ?? 'UNKNOWN'; // 未识别
}
/**
* 二手信息入库
*/
public function saveUsedcarBasicToDb(array $data = [])
{
$data['created_at'] = time();
$data['updated_at'] = time();
return Db::table('usedcar_crawl_results')->insertGetId($data);
}
/**
* 更新信息入库
*/
public function saveUsedcarConfigToDb(array $data = [], int $id = 0)
{
$info = Db::table('usedcar_crawl_results')->find($id);
if (!$info) {
return false;
}
$groups = $data['groups'];
$formatCarInfo = array_merge($data['format_car_info'], json_decode($info->format_car_info, true));
return Db::table('usedcar_crawl_results')
->where('id', $id)
->update([
'format_car_info' => json_encode($formatCarInfo, JSON_UNESCAPED_UNICODE),
'config_detail' => json_encode($groups, JSON_UNESCAPED_UNICODE),
'updated_at' => time()
]);
}
/**
* 把 car_info 扁平成一个 [key => ['value' => xxx]] 的 map
*
* @param array $carInfo rawData['car_info'][0]
* @return array
*/
public function buildValuesMapFromCarInfo(array $carInfo): array
{
$valuesMap = [];
// 1) car_info 顶层的一些字段（official_price、series_name 等）
foreach ($carInfo as $k => $v) {
if ($k === 'info' || is_array($v)) {
continue;
}
$valuesMap[$k] = ['value' => $v];
}
// 2) info 里的配置项
if (!empty($carInfo['info']) && is_array($carInfo['info'])) {
foreach ($carInfo['info'] as $k => $v) {
if (is_array($v) && array_key_exists('value', $v)) {
$valuesMap[$k] = $v;
}
}
}
return $valuesMap;
}
/**
* 根据 type 重新分组：
* - type=0,2 为分组（左侧分类）
* - 其它 type 都视为该分组下的字段
*
* @param array $properties rawData['properties']
* @param array $valuesMap 来自 buildValuesMapFromCarInfo()
* @return array
*/
public function buildCarSpecGroups(array $properties, array $valuesMap): array
{
$groups = [];
$currentKey = null; // 当前分组 key
foreach ($properties as $item) {
$type = (int)($item['type'] ?? 1);
$key = $item['key'] ?? '';
$text = $item['text'] ?? '';
$subList = $item['sub_list'] ?? null;
// ---------- 1) 分类字段：type = 0 / 2 ----------
if (in_array($type, [0, 2], true)) {
$currentKey = $key ?: ($text ?: ('group_' . count($groups)));
if (!isset($groups[$currentKey])) {
$groups[$currentKey] = [
'group_key' => $currentKey,
'group_text' => $text,
'group_type' => $type,
'items' => [],
];
}
continue;
}
// ---------- 2) 普通字段：挂到最近的分类下 ----------
if ($currentKey === null) {
// 如果一开始就没有遇到分类，给一个默认分组
$currentKey = '_default';
$groups[$currentKey] = [
'group_key' => '_default',
'group_text' => '其他',
'group_type' => -1,
'items' => [],
];
}
// 取 value
$value = '';
if ($key && isset($valuesMap[$key])) {
$value = trim((string)($valuesMap[$key]['value'] ?? ''));
}
$node = [
'key' => $key,
'text' => $text,
'value' => $value,
];
// ---------- 3) sub_list 子项（有就当成 children） ----------
if (is_array($subList) && !empty($subList)) {
$children = [];
foreach ($subList as $sub) {
$subKey = $sub['key'] ?? '';
$subText = $sub['text'] ?? '';
$subValue = '';
if ($subKey && isset($valuesMap[$subKey])) {
$subValue = trim((string)($valuesMap[$subKey]['value'] ?? ''));
}
$children[] = [
'key' => $subKey,
'text' => $subText,
'value' => $subValue,
];
}
$node['children'] = $children;
}
$groups[$currentKey]['items'][] = $node;
}
// 你可以按需返回关联数组或索引数组，这里转成索引数组方便前端遍历
return array_values($groups);
}
/**
* 转义价格
*/
protected function decodeDongchediPrice(string $str): string
{
// 映射表：PUA 字符 => 数字 / 万
$map = [
"\u{E53D}" => '1',
"\u{E3F0}" => '2',
"\u{E422}" => '3',
"\u{E42C}" => '4',
"\u{E49C}" => '5',
"\u{E42B}" => '6',
"\u{E4FE}" => '7',
"\u{E548}" => '8',
"\u{E4C8}" => '9',
"\u{E453}" => '0',
"\u{E45F}" => '万',
];
return strtr($str, $map);
}
/**
* 获取URL最后ID
*/
protected function getLastIdFromUrl($url)
{
// 方法1：使用正则表达式提取最后的数字ID
if (preg_match('/\/(\d+)$/', $url, $matches)) {
return $matches[1];
}
return false;
}
/**
* 下载单张图片
*
* @param string $url 图片完整 URL
* @param string $saveDir 保存目录（如 /www/usedcar_images/2025-11-20）
* @param string|null $filename 指定文件名（可选，不含路径），为空则自动生成
* @param array $headers 额外 HTTP 头（可选）
*
* @return string|null 成功返回保存路径，失败返回 null
*/
protected function downloadImage(string $url, string $saveDir, ?string $filename = null, array $headers = []): ?string
{
// 1. 目录检查 & 创建
if (!is_dir($saveDir)) {
if (!mkdir($saveDir, 0777, true) && !is_dir($saveDir)) {
echo "[Image] 创建目录失败：{$saveDir}\n";
return null;
}
}
// 2. 自动生成文件名（基于 URL md5 + 原扩展名）
if ($filename === null) {
$path = parse_url($url, PHP_URL_PATH); // /xxx/xxx.jpg
$ext = pathinfo($path, PATHINFO_EXTENSION); // jpg 或 webp 之类
if ($ext === '') {
$ext = 'jpg'; // 没有扩展名时兜底
}
$filename = md5($url) . '.' . $ext;
}
$savePath = rtrim($saveDir, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . $filename;
// 3. 组装 HTTP 头
$defaultHeaders = [
'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
. 'AppleWebKit/537.36 (KHTML, like Gecko) '
. 'Chrome/120.0.0.0 Safari/537.36',
'Accept: image/avif,image/webp,image/apng,image/*,*/*;q=0.8',
'Referer: https://www.dongchedi.com/', // 很多图床需要 Referer
];
foreach ($headers as $h) {
$defaultHeaders[] = $h;
}
// 4. 使用 cURL 下载
$ch = curl_init($url);
curl_setopt_array($ch, [
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_CONNECTTIMEOUT => 5,
CURLOPT_TIMEOUT => 15,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_SSL_VERIFYHOST => false,
CURLOPT_HTTPHEADER => $defaultHeaders,
]);
$data = curl_exec($ch);
$errNo = curl_errno($ch);
$errMsg = curl_error($ch);
$code = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE);
curl_close($ch);
if ($errNo !== 0) {
echo "[Image] 下载失败(cURL)：{$url} | {$errNo} {$errMsg}\n";
return null;
}
if ($code !== 200 || $data === false || $data === '') {
echo "[Image] 下载失败(HTTP {$code})：{$url}\n";
return null;
}
// 5. 写入本地文件
$bytes = file_put_contents($savePath, $data);
if ($bytes === false) {
echo "[Image] 保存失败：{$savePath}\n";
return null;
}
echo "[Image] 下载成功：{$url} => {$savePath}\n";
return $savePath;
}
/**
* 批量下载相册
*
* @param array $urls 图片 URL 数组
* @param string $saveDir 保存目录
*
* @return array [原始 URL => 保存路径或 null]
*/
protected function downloadImageAlbum(array $urls, string $saveDir): array
{
$result = [];
foreach ($urls as $url) {
$url = trim($url);
if ($url === '') {
continue;
}
// 如果存在 // 开头的相对协议，补全
if (strpos($url, '//') === 0) {
$url = 'https:' . $url;
}
$path = $this->downloadImage($url, $saveDir);
$result[$url] = $path;
}
return $result;
}
}
app/controller/IndexController.php
+11 -4
View File
@@ -159,9 +159,16 @@ class IndexController
public function test(Request $request)
{
$result = [
'desc' => 'demo'
];
return json(['code' => 0, 'msg' => 'ok', 'data' => $result]);
$service = new \app\common\service\UsedCarCrawlService();
$result = $service->iUrl('https://www.dongchedi.com/usedcar/21557932');
// $result = $service->doData();
// $result = $service->decodeDongchediPrice('.');
// $result = $service->classifyEnergy('插电式混合动力');
return json($result);
}
}
app/process/CheckWeb.php
+6
View File
@@ -22,6 +22,12 @@ class CheckWeb
// });
// }
// 每秒执行：队列二手车数据处理
new Crontab('* * * * * *', function(){
$service = new \app\common\service\UsedCarCrawlService();
$service->doData();
});
new Crontab('*/50 * * * * *', function(){
$userIds = User::where('status', 1)->pluck('id')->toArray();
foreach ($userIds as $userId) {