Files
useds/app/common/utils/crawl/service/UsedCarCrawl_bak.php
T
2026-01-05 14:33:21 +08:00

682 lines
23 KiB
PHP
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<?php
namespace app\common\utils\crawl\service;
use GuzzleHttp\Client;
use QL\QueryList;
use support\Redis;
use support\Db;
use support\exception\BusinessException;
class UsedCarCrawl_bak
{
/**
* 插入队列url
*
* $scene 二手信息 usedcar 配置信息 config
*/
public function iUrl(string $url = '', string $scene = 'usedcar', int $id = 0)
{
return Redis::lpush('usedcar:urls', json_encode([
'scene' => $scene,
'url' => $url,
'id' => $id
], JSON_UNESCAPED_UNICODE));
}
/**
* 获取队列url
*
* $scene 二手信息 usedcar 配置信息 config
*/
public function rUrl()
{
return json_decode(Redis::rPop('usedcar:urls'), true);
}
/**
* 执行抓取
*/
public function doData()
{
try {
$raw = $this->rUrl();
if (!$raw) {
return false;
}
$html = $this->getHtml($raw['url']);
switch ($raw['scene']) {
case 'usedcar':
$info = $this->getUsedDetail($html, $raw['url']);
$id = $this->saveUsedcarBasicToDb($info);
$this->iUrl($info['car_id_url'], 'config', $id);
break;
case 'config':
$data = $this->getConfigDetail($html);
$this->saveUsedcarConfigToDb($data, $raw['id']);
break;
}
return true;
} catch (\Throwable $th) {
throw new BusinessException($th->getMessage().'|'.$th->getLine().'|'.$th->getFile());
}
}
/**
* 获取html
*/
public function getHtml($url)
{
$client = new Client();
$res = $client->request('GET', $url, [
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
. 'AppleWebKit/537.36 (KHTML, like Gecko) '
. 'Chrome/120.0.0.0 Safari/537.36',
'Accept-Language' => 'zh-CN,zh;q=0.9',
'Referer' => 'https://www.dongchedi.com/'
]
]);
return (string)$res->getBody();
}
/**
* 获取二手信息
*/
public function getUsedDetail(string $html = '', string $url = '')
{
$ql = QueryList::html($html);
$title = $ql->find('h1')->text();
$price = $ql->find('.head-info_price-wrap__Y4bxi > p')->text();
$pageText = $ql->find('body')->text();
$jsonText = $ql->find('#__NEXT_DATA__')->text();
// 新车指导价
$newPrice = '';
if ($pageText) {
if (preg_match('/新车指导价[::]\s*([^\r\n万]*万)/u', $pageText, $m)) {
$newPrice = trim(preg_replace('/[^0-9.]/', '', $m[1]));
}
}
// 比新车省
$priceDifference = '';
if ($pageText) {
if (preg_match('/比新车省[::]\s*([^\r\n万]*万)/u', $pageText, $m)) {
$priceDifference = trim(preg_replace('/[^0-9.]/', '', $m[1]));
}
}
// 车况介绍
$conditionDesc = '';
if ($pageText) {
if (preg_match('/车况介绍\s*(.*?)\s*推荐理由/su', $pageText, $m)) {
$conditionDesc = trim($m[1]);
}
}
$jsonArray = json_decode($jsonText, true);
// 车辆相册
$images = $jsonArray['props']['pageProps']['skuDetail']['head_images'] ?? [];
// 车辆ID
$carId = $jsonArray['props']['pageProps']['skuDetail']['car_info']['car_id'] ?? 0;
// 车辆年份
$year = $jsonArray['props']['pageProps']['skuDetail']['car_info']['year'] ?? 0;
// 车辆ID URL
$carIdUrl = $carId ? ('https://www.dongchedi.com/auto/params-carIds-' . $carId) : '';
// 能源类型
$fuelForm = $jsonArray['props']['pageProps']['skuDetail']['car_config_overview']['power']['fuel_form'] ?? '';
$newFuelForm = $jsonArray['props']['pageProps']['skuDetail']['car_config_overview']['new_energy_power']['fuel_form'] ?? '';
$resultFuelForm = $fuelForm === '' ? $newFuelForm : $fuelForm;
// 使用里程
$importantText = $jsonArray['props']['pageProps']['skuDetail']['important_text'] ?? '';
$importantTextArray = explode(' | ', $importantText);
$usageMileage = str_replace('万公里', '', isset($importantTextArray[1]) ? $importantTextArray[1] : '');
// 车辆名称 品牌、车系、车型
$brandName = $jsonArray['props']['pageProps']['skuDetail']['car_info']['brand_name'] ?? '';
$seriesName = $jsonArray['props']['pageProps']['skuDetail']['car_info']['series_name'] ?? '';
$carName = $jsonArray['props']['pageProps']['skuDetail']['car_info']['car_name'] ?? '';
// 商家信息
$shopInfo = $jsonArray['props']['pageProps']['skuDetail']['shop_info'] ?? [];
$formatShopInfo = [
'shop_id' => $shopInfo['shop_id'] ?? 0,
'shop_name' => $shopInfo['shop_short_name'] ?? '',
'shop_address' => $shopInfo['shop_address'] ?? ''
];
// 简介参数
$otherParams = $jsonArray['props']['pageProps']['skuDetail']['other_params'] ?? [];
// 排量
$displacement = $this->getOtherParamValue($otherParams, '排量');
// 变速箱
$gearbox = $this->getOtherParamValue($otherParams, '变速箱');
// 格式化车辆信息
$formatCarInfo = [
'displacement' => $displacement,
'gearbox' => $gearbox
];
// 整理车辆信息
$info['used_car_id'] = $this->getLastIdFromUrl($url);
$info['car_id'] = $carId;
$info['url'] = $url;
$info['title'] = $title;
$info['price'] = str_replace('万', '', $this->decodeDongchediPrice($price));
$info['new_price'] = str_replace('万', '', $newPrice);
$info['price_difference'] = str_replace('万', '', $priceDifference);
$info['condition_desc'] = $conditionDesc;
$info['images'] = json_encode($images);
$info['car_id_url'] = $carIdUrl;
$info['used_detail'] = $jsonText;
$info['format_shop_info'] = json_encode($formatShopInfo);
$info['fuel_form'] = $resultFuelForm;
$info['classify_energy'] = $this->detectEnergyType($resultFuelForm);
$info['usage_mileage'] = $usageMileage;
$info['brand_name'] = $brandName;
$info['series_name'] = $seriesName;
$info['car_name'] = $carName;
$info['year'] = $year;
$info['format_car_info'] = json_encode($formatCarInfo);
return $info;
}
/**
* 获取配置信息
*/
public function getConfigDetail(string $html = '')
{
$ql = QueryList::html($html);
$jsonText = $ql->find('#__NEXT_DATA__')->text();
$raw = json_decode($jsonText, true);
$properties = $raw['props']['pageProps']['rawData']['properties'] ?? [];
$carInfo = $raw['props']['pageProps']['rawData']['car_info'][0] ?? [];
$valuesMap = $this->buildValuesMapFromCarInfo($carInfo);
$groups = $this->buildCarSpecGroups($properties, $valuesMap);
$data['format_car_info'] = [
'market_time' => $this->getFieldValue($groups, 'market_time'), // 上市时间
'driver_form' => $this->getFieldValue($groups, 'driver_form'), // 驱动形式
'seat_count' => $this->getFieldValue($groups, 'seat_count'), // 座位数
'engine_description' => $this->getFieldValue($groups, 'engine_description'), // 发动机
'engine_max_power_horsepower' => $this->getFieldValue($groups, 'engine_max_power_horsepower'), // 发动机最大功率
'electric_description' => $this->getFieldValue($groups, 'electric_description'), //电动机
'battery_capacity' => $this->getFieldValue($groups, 'battery_capacity'), //电池容量
'battery_type' => $this->getFieldValue($groups, 'battery_type'), // 电池类型
'recharge_mileage' => $this->getFieldValue($groups, 'recharge_mileage') // 续航里程
];
$data['groups'] = $groups;
return $data;
}
/**
* 从车辆配置 JSON 数据中获取能源类型
*
* @param array $data JSON 数据
* @return string|null 能源类型,如 '油电混合'、'纯电动'、
*/
public function getFuelFormByConfigInfo(array $data)
{
$fuelForm = null;
foreach ($data[0]['items'] as $item) {
if ($item['key'] === 'fuel_form') {
$fuelForm = $item['value'];
break;
}
}
return $fuelForm;
}
/**
* 从车辆配置 JSON 数据中根据字段 key 获取对应的 value
*
* @param string|array $data JSON 字符串 或 已解析的数组
* @param string $field 要查找的字段 key(如 'fuel_form')
* @return string|null 找到则返回 value,否则返回 null
*/
public function getFieldValue($data, string $field)
{
// 如果传入的是 JSON 字符串,先解码
if (is_string($data)) {
$data = json_decode($data, true);
if (json_last_error() !== JSON_ERROR_NONE) {
return null; // JSON 格式错误
}
}
// 遍历第一层(假设结构是 [ { "items": [...] } ])
foreach ($data as $group) {
if (isset($group['items']) && is_array($group['items'])) {
foreach ($group['items'] as $item) {
if (isset($item['key']) && $item['key'] === $field) {
return $item['value'] ?? null;
}
}
}
}
return null; // 未找到
}
/**
* 从 other_params 数组中根据 name 获取对应的 value
*
* @param array $otherParams 格式为 [ ['name' => 'xxx', 'value' => 'yyy'], ... ]
* @param string $name 要查找的字段名称(如 '上牌地')
* @return string|null 找到则返回 value,否则返回 null
*/
public function getOtherParamValue(array $otherParams, string $name): ?string
{
foreach ($otherParams as $item) {
if (isset($item['name']) && $item['name'] === $name) {
return $item['value'] ?? null;
}
}
return null;
}
/**
* 根据给定字符串判断车辆能源类型
*
* 返回:
* - RE 新能源(纯电、插电混动、增程、氢燃料等)
* - CE 传统能源(汽油、柴油、天然气、非插电混动等)
* - UNKNOWN 无法判断
*
* @param string $text 例如:'2022款 某某牌 2.0L 油电混合 双擎'
* @return string
*/
public function detectEnergyType(string $text): string
{
if (function_exists('mb_internal_encoding')) {
mb_internal_encoding('UTF-8');
}
// 明确定义的新能源关键字(纯电 / 插电 / 增程 / 氢 / 太阳能 等)
$newEnergyKeywords = [
// 纯电
'纯电', '纯电动', '电动车', 'BEV',
// 插电式混合
'插电混动', '插电式混合', '插电式', '插电', 'PHEV',
// 增程式
'增程式', '增程', 'REEV',
// 氢能 / 燃料电池
'氢能', '氢燃料', '燃料电池', 'FCEV',
// 其它
'太阳能',
];
// 传统能源关键字(包括燃油/燃气)
$traditionalKeywords = [
'汽油',
'柴油',
'天然气', 'LNG', 'CNG',
];
// 普通混动关键字(按你的要求,统一归为传统能源)
$hybridKeywords = [
'油电混合', '油电混动', '油电',
'混动', // 不带“插电”的混动
'双擎', // 丰田常见表达
'HEV',
];
// 辅助:不区分大小写的包含判断,优先 mb_stripos
$contains = function (string $haystack, string $needle): bool {
if ($needle === '') {
return false;
}
if (function_exists('mb_stripos')) {
return mb_stripos($haystack, $needle) !== false;
}
return stripos($haystack, $needle) !== false;
};
// 1. 先判断“明确新能源”关键词(插电、增程、氢、纯电等)
foreach ($newEnergyKeywords as $kw) {
if ($contains($text, $kw)) {
return 'RE';
}
}
// 2. 再判断“普通油电混动”——按照你的业务,一律归为传统能源
foreach ($hybridKeywords as $kw) {
if ($contains($text, $kw)) {
return 'CE';
}
}
// 3. 再判断传统能源(纯燃油/气)
foreach ($traditionalKeywords as $kw) {
if ($contains($text, $kw)) {
return 'CE';
}
}
// 4. 都没匹配上
return 'UNKNOWN';
}
/**
* 二手信息入库
*/
public function saveUsedcarBasicToDb(array $data = [])
{
$data['created_at'] = time();
$data['updated_at'] = time();
return Db::table('usedcar_crawl_results')->insertGetId($data);
}
/**
* 更新信息入库
*/
public function saveUsedcarConfigToDb(array $data = [], int $id = 0)
{
$info = Db::table('usedcar_crawl_results')->find($id);
if (!$info) {
return false;
}
$groups = $data['groups'];
$formatCarInfo = array_merge($data['format_car_info'], json_decode($info->format_car_info, true));
return Db::table('usedcar_crawl_results')
->where('id', $id)
->update([
'format_car_info' => json_encode($formatCarInfo, JSON_UNESCAPED_UNICODE),
'config_detail' => json_encode($groups, JSON_UNESCAPED_UNICODE),
'updated_at' => time()
]);
}
/**
* 把 car_info 扁平成一个 [key => ['value' => xxx]] 的 map
*
* @param array $carInfo rawData['car_info'][0]
* @return array
*/
public function buildValuesMapFromCarInfo(array $carInfo): array
{
$valuesMap = [];
// 1) car_info 顶层的一些字段(official_price、series_name 等)
foreach ($carInfo as $k => $v) {
if ($k === 'info' || is_array($v)) {
continue;
}
$valuesMap[$k] = ['value' => $v];
}
// 2) info 里的配置项
if (!empty($carInfo['info']) && is_array($carInfo['info'])) {
foreach ($carInfo['info'] as $k => $v) {
if (is_array($v) && array_key_exists('value', $v)) {
$valuesMap[$k] = $v;
}
}
}
return $valuesMap;
}
/**
* 根据 type 重新分组:
* - type=0,2 为分组(左侧分类)
* - 其它 type 都视为该分组下的字段
*
* @param array $properties rawData['properties']
* @param array $valuesMap 来自 buildValuesMapFromCarInfo()
* @return array
*/
public function buildCarSpecGroups(array $properties, array $valuesMap): array
{
$groups = [];
$currentKey = null; // 当前分组 key
foreach ($properties as $item) {
$type = (int)($item['type'] ?? 1);
$key = $item['key'] ?? '';
$text = $item['text'] ?? '';
$subList = $item['sub_list'] ?? null;
// ---------- 1) 分类字段:type = 0 / 2 ----------
if (in_array($type, [0, 2], true)) {
$currentKey = $key ?: ($text ?: ('group_' . count($groups)));
if (!isset($groups[$currentKey])) {
$groups[$currentKey] = [
'group_key' => $currentKey,
'group_text' => $text,
'group_type' => $type,
'items' => [],
];
}
continue;
}
// ---------- 2) 普通字段:挂到最近的分类下 ----------
if ($currentKey === null) {
// 如果一开始就没有遇到分类,给一个默认分组
$currentKey = '_default';
$groups[$currentKey] = [
'group_key' => '_default',
'group_text' => '其他',
'group_type' => -1,
'items' => [],
];
}
// 取 value
$value = '';
if ($key && isset($valuesMap[$key])) {
$value = trim((string)($valuesMap[$key]['value'] ?? ''));
}
$node = [
'key' => $key,
'text' => $text,
'value' => $value,
];
// ---------- 3) sub_list 子项(有就当成 children) ----------
if (is_array($subList) && !empty($subList)) {
$children = [];
foreach ($subList as $sub) {
$subKey = $sub['key'] ?? '';
$subText = $sub['text'] ?? '';
$subValue = '';
if ($subKey && isset($valuesMap[$subKey])) {
$subValue = trim((string)($valuesMap[$subKey]['value'] ?? ''));
}
$children[] = [
'key' => $subKey,
'text' => $subText,
'value' => $subValue,
];
}
$node['children'] = $children;
}
$groups[$currentKey]['items'][] = $node;
}
// 你可以按需返回关联数组或索引数组,这里转成索引数组方便前端遍历
return array_values($groups);
}
/**
* 转义价格
*/
protected function decodeDongchediPrice(string $str): string
{
// 映射表:PUA 字符 => 数字 / 万
$map = [
"\u{E53D}" => '1',
"\u{E3F0}" => '2',
"\u{E422}" => '3',
"\u{E42C}" => '4',
"\u{E49C}" => '5',
"\u{E42B}" => '6',
"\u{E4FE}" => '7',
"\u{E548}" => '8',
"\u{E4C8}" => '9',
"\u{E453}" => '0',
"\u{E45F}" => '万',
];
return strtr($str, $map);
}
/**
* 获取URL最后ID
*/
protected function getLastIdFromUrl($url)
{
// 方法1:使用正则表达式提取最后的数字ID
if (preg_match('/\/(\d+)$/', $url, $matches)) {
return $matches[1];
}
return false;
}
/**
* 下载单张图片
*
* @param string $url 图片完整 URL
* @param string $saveDir 保存目录(如 /www/usedcar_images/2025-11-20)
* @param string|null $filename 指定文件名(可选,不含路径),为空则自动生成
* @param array $headers 额外 HTTP 头(可选)
*
* @return string|null 成功返回保存路径,失败返回 null
*/
protected function downloadImage(string $url, string $saveDir, ?string $filename = null, array $headers = []): ?string
{
// 1. 目录检查 & 创建
if (!is_dir($saveDir)) {
if (!mkdir($saveDir, 0777, true) && !is_dir($saveDir)) {
echo "[Image] 创建目录失败:{$saveDir}\n";
return null;
}
}
// 2. 自动生成文件名(基于 URL md5 + 原扩展名)
if ($filename === null) {
$path = parse_url($url, PHP_URL_PATH); // /xxx/xxx.jpg
$ext = pathinfo($path, PATHINFO_EXTENSION); // jpg 或 webp 之类
if ($ext === '') {
$ext = 'jpg'; // 没有扩展名时兜底
}
$filename = md5($url) . '.' . $ext;
}
$savePath = rtrim($saveDir, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . $filename;
// 3. 组装 HTTP 头
$defaultHeaders = [
'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
. 'AppleWebKit/537.36 (KHTML, like Gecko) '
. 'Chrome/120.0.0.0 Safari/537.36',
'Accept: image/avif,image/webp,image/apng,image/*,*/*;q=0.8',
'Referer: https://www.dongchedi.com/', // 很多图床需要 Referer
];
foreach ($headers as $h) {
$defaultHeaders[] = $h;
}
// 4. 使用 cURL 下载
$ch = curl_init($url);
curl_setopt_array($ch, [
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_CONNECTTIMEOUT => 5,
CURLOPT_TIMEOUT => 15,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_SSL_VERIFYHOST => false,
CURLOPT_HTTPHEADER => $defaultHeaders,
]);
$data = curl_exec($ch);
$errNo = curl_errno($ch);
$errMsg = curl_error($ch);
$code = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE);
curl_close($ch);
if ($errNo !== 0) {
echo "[Image] 下载失败(cURL):{$url} | {$errNo} {$errMsg}\n";
return null;
}
if ($code !== 200 || $data === false || $data === '') {
echo "[Image] 下载失败(HTTP {$code}):{$url}\n";
return null;
}
// 5. 写入本地文件
$bytes = file_put_contents($savePath, $data);
if ($bytes === false) {
echo "[Image] 保存失败:{$savePath}\n";
return null;
}
echo "[Image] 下载成功:{$url} => {$savePath}\n";
return $savePath;
}
/**
* 批量下载相册
*
* @param array $urls 图片 URL 数组
* @param string $saveDir 保存目录
*
* @return array [原始 URL => 保存路径或 null]
*/
protected function downloadImageAlbum(array $urls, string $saveDir): array
{
$result = [];
foreach ($urls as $url) {
$url = trim($url);
if ($url === '') {
continue;
}
// 如果存在 // 开头的相对协议,补全
if (strpos($url, '//') === 0) {
$url = 'https:' . $url;
}
$path = $this->downloadImage($url, $saveDir);
$result[$url] = $path;
}
return $result;
}
}