Files
x-server-mobile/backend/app/service/UsedCarCrawlService.php
T
2026-03-05 16:21:22 +08:00

683 lines
22 KiB
PHP
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<?php
namespace app\Service;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException;
use QL\QueryList;
use support\Redis;
/**
* 二手车数据抓取服务
* 支持从懂车帝、汽车之家等网站抓取车辆信息
*/
class UsedCarCrawlService
{
private Client $httpClient;
// 默认请求头
private const DEFAULT_HEADERS = [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept-Language' => 'zh-CN,zh;q=0.9',
'Referer' => 'https://www.dongchedi.com/',
];
public function __construct()
{
$this->httpClient = new Client([
'verify' => false,
'timeout' => 15,
]);
}
/**
* 选择抓取方式并执行
*
* @param string $url 车辆详情页URL
* @return array 抓取结果
* @throws \Exception
*/
public function crawlByUrl(string $url): array
{
// 根据URL判断来源并选择抓取方法
if (str_contains($url, 'dongchedi.com')) {
// 懂车帝PC端
return $this->crawlDongchedi($url);
} elseif (str_contains($url, 'dcdapp.com')) {
// 懂车帝移动端 - 需要先转换为PC端
$pcUrl = $this->getDongchediPcUrl($url);
if (!$pcUrl) {
throw new \Exception('无法获取懂车帝PC端链接');
}
return $this->crawlDongchedi($pcUrl);
} elseif (str_contains($url, 't.zijieimg.com')) {
// 懂车帝分享短链
$finalUrl = $this->getFinalUrl($url);
$pcUrl = $this->getDongchediPcUrl($finalUrl);
if (!$pcUrl) {
throw new \Exception('无法获取懂车帝PC端链接');
}
return $this->crawlDongchedi($pcUrl);
} elseif (str_contains($url, 'che168.com')) {
// 汽车之家
return $this->crawlChe168($url);
}
throw new \Exception('暂不支持此网站的数据抓取');
}
/**
* 获取短链的最终URL
*/
private function getFinalUrl(string $url): string
{
try {
$response = $this->httpClient->get($url, [
'allow_redirects' => ['track_redirects' => true],
]);
$history = $response->getHeader('X-Guzzle-Redirect-History');
return end($history) ?: $url;
} catch (\Exception $e) {
throw new \Exception('获取短链失败: ' . $e->getMessage());
}
}
/**
* 从懂车帝移动端URL获取PC端URL
*/
private function getDongchediPcUrl(string $url): ?string
{
$query = parse_url($url, PHP_URL_QUERY);
if (!$query) {
return null;
}
parse_str($query, $params);
$skuId = $params['sku_id'] ?? null;
return $skuId ? 'https://www.dongchedi.com/usedcar/' . $skuId : null;
}
/**
* 抓取懂车帝二手车信息
*/
private function crawlDongchedi(string $url): array
{
try {
$html = $this->fetchHtml($url);
if (str_contains($html, '404 Not Found')) {
throw new \Exception('车辆已下架或找不到页面');
}
return $this->parseDongchediHtml($html, $url);
} catch (\Exception $e) {
throw new \Exception('抓取懂车帝数据失败: ' . $e->getMessage());
}
}
/**
* 抓取汽车之家二手车信息
*/
private function crawlChe168(string $url): array
{
try {
// 处理不同格式的汽车之家URL
if (str_contains($url, 'home?infoid=')) {
// PC端链接转换
$queryString = parse_url($url, PHP_URL_QUERY);
parse_str($queryString, $params);
$carId = $params['infoid'] ?? null;
$url = 'https://www.che168.com/dealer/111/' . $carId . '.html?pvareaid=0';
} elseif (str_contains($url, 'wap.che168.com/dealer')) {
// 移动端链接转换
$url = $this->getChe168PcUrl($url);
}
$html = $this->fetchHtml($url);
return $this->parseChe168Html($html, $url);
} catch (\Exception $e) {
throw new \Exception('抓取汽车之家数据失败: ' . $e->getMessage());
}
}
/**
* 从汽车之家移动端URL获取PC端URL
*/
private function getChe168PcUrl(string $url): string
{
$path = parse_url($url, PHP_URL_PATH);
$parts = explode('/', $path);
$filename = end($parts);
$carId = pathinfo($filename, PATHINFO_FILENAME);
return 'https://www.che168.com/dealer/111/' . $carId . '.html?pvareaid=0';
}
/**
* 获取网页HTML内容
*/
private function fetchHtml(string $url): string
{
try {
$response = $this->httpClient->get($url, [
'headers' => self::DEFAULT_HEADERS,
]);
return (string)$response->getBody();
} catch (GuzzleException $e) {
throw new \Exception('获取页面失败: ' . $e->getMessage());
}
}
/**
* 解析懂车帝HTML
*/
private function parseDongchediHtml(string $html, string $url): array
{
try {
$ql = QueryList::html($html);
// 提取JSON数据
$jsonText = $ql->find('#__NEXT_DATA__')->text();
if (empty($jsonText)) {
throw new \Exception('无法找到页面数据');
}
$jsonData = json_decode($jsonText, true);
if (!$jsonData) {
throw new \Exception('解析JSON数据失败');
}
$skuDetail = $jsonData['props']['pageProps']['skuDetail'] ?? [];
$carInfo = $skuDetail['car_info'] ?? [];
// 提取价格(懂车帝使用特殊字符编码)
$priceText = $ql->find('.head-info_price-wrap__Y4bxi > p')->text();
$price = $this->decodeDongchediPrice($priceText);
// 提取车辆信息
$title = $carInfo['car_name'] ?? '';
$brandName = $carInfo['brand_name'] ?? '';
$seriesName = $carInfo['series_name'] ?? '';
$year = $carInfo['year'] ?? '';
$bodyColor = $carInfo['body_color'] ?? '';
// 提取配置信息
$overview = $skuDetail['car_config_overview'] ?? [];
$power = $overview['power']['horsepower'] ?? '';
$displacement = $overview['power']['capacity'] ?? '';
// 提取上牌时间和车源地
$otherParams = $skuDetail['other_params'] ?? [];
$plateTime = $this->getParamValue($otherParams, '上牌时间');
$sourcePlace = $this->getParamValue($otherParams, '车源地');
// 提取里程
$importantText = $skuDetail['important_text'] ?? '';
$mileage = 0;
if (preg_match('/(\d+\.?\d*)\s*万公里/', $importantText, $matches)) {
$mileage = (int)($matches[1] * 10000);
}
// 提取图片
$images = $skuDetail['head_images'] ?? [];
// 提取车况介绍
$conditionDesc = '';
if (preg_match('/车况介绍\s*(.*?)\s*推荐理由/s', $html, $matches)) {
$conditionDesc = trim($matches[1]);
}
// 提取车辆ID
$carId = $carInfo['car_id'] ?? 0;
$brandId = $carInfo['brand_id'] ?? 0;
$seriesId = $carInfo['series_id'] ?? 0;
$usedCarId = $this->extractIdFromUrl($url);
return [
'title' => $title,
'price' => $price,
'brand_name' => $brandName,
'brand_id' => $brandId,
'series_name' => $seriesName,
'series_id' => $seriesId,
'year' => $year,
'car_id' => $carId,
'used_car_id' => $usedCarId,
'power' => $power,
'displacement' => $displacement,
'plate_time' => $plateTime,
'source_place' => $sourcePlace,
'body_color' => $bodyColor,
'mileage' => $mileage,
'images' => $images,
'condition_desc' => $conditionDesc,
'source_url' => $url,
];
} catch (\Exception $e) {
throw new \Exception('解析懂车帝页面失败: ' . $e->getMessage());
}
}
/**
* 解析汽车之家HTML
*/
private function parseChe168Html(string $html, string $url): array
{
try {
// 汽车之家使用 GB2312 编码,需要转换
$html = $this->convertToUtf8($html);
$ql = QueryList::html($html);
// 提取车辆名称
$productName = $ql->find('div.car-box .car-brand-name')->text();
// 提取价格
$price = $ql->find('div.goodstartmoney')->text();
if (empty($price)) {
$price = $ql->find('span#overlayPrice')->text();
}
$price = str_replace(['¥', '¥', '万'], '', $price);
$price = is_numeric($price) ? (int)($price * 10000) : 0;
// 提取基本信息
$basicInfo = [];
$ql->find('div#nav1 ul.basic-item-ul li')->each(function($li) use (&$basicInfo) {
$field = trim($li->find('.item-name')->text());
$value = trim(str_replace($field, '', $li->text()));
$fieldMap = [
'上牌时间' => 'plate_time',
'表显里程' => 'mileage_text',
'排       量' => 'displacement',
'所  在  地' => 'source_place',
'车身颜色' => 'body_color',
'车况介绍' => 'condition_desc',
];
if (isset($fieldMap[$field])) {
$basicInfo[$fieldMap[$field]] = $value;
}
});
// 处理里程
$mileage = 0;
if (isset($basicInfo['mileage_text'])) {
$mileage = (int)str_replace('万公里', '', $basicInfo['mileage_text']) * 10000;
}
// 提取面包屑导航获取品牌和车系
$breadcrumbs = [];
$ql->find('.bread-crumbs a')->each(function($a) use (&$breadcrumbs) {
$breadcrumbs[] = trim($a->text());
});
$brandName = $breadcrumbs[3] ?? '';
$seriesName = $breadcrumbs[4] ?? '';
$brandName = str_replace('二手', '', $brandName);
$seriesName = str_replace('二手', '', $seriesName);
// 提取年份
$year = '';
if (preg_match('/(\d{4})款/', $productName, $matches)) {
$year = $matches[1];
}
// 提取图片
$images = [];
$ql->find('.swiper-container2 .swiper-slide .jiaodianphotoclick img')->each(function($img) use (&$images) {
$src = $img->attr('src');
if ($src && !str_contains($src, 'default-che168')) {
$images[] = $src;
}
});
// 提取车辆ID
$usedCarId = $this->extractIdFromUrl($url);
return [
'title' => $productName,
'product_name' => $productName,
'price' => $price,
'brand_name' => $brandName,
'brand_id' => 0,
'series_name' => $seriesName,
'series_id' => 0,
'year' => $year,
'car_id' => 0,
'used_car_id' => $usedCarId,
'plate_time' => $basicInfo['plate_time'] ?? '',
'displacement' => $basicInfo['displacement'] ?? '',
'source_place' => $basicInfo['source_place'] ?? '',
'body_color' => $basicInfo['body_color'] ?? '',
'mileage' => $mileage,
'images' => $images,
'condition_desc' => $basicInfo['condition_desc'] ?? '',
'source_url' => $url,
];
} catch (\Exception $e) {
throw new \Exception('解析汽车之家页面失败: ' . $e->getMessage());
}
}
/**
* 解码懂车帝价格(特殊字符编码)
*/
private function decodeDongchediPrice(string $str): float
{
$map = [
"\u{E53D}" => '1',
"\u{E3F0}" => '2',
"\u{E422}" => '3',
"\u{E42C}" => '4',
"\u{E49C}" => '5',
"\u{E42B}" => '6',
"\u{E4FE}" => '7',
"\u{E548}" => '8',
"\u{E4C8}" => '9',
"\u{E453}" => '0',
"\u{E45F}" => '.',
];
$priceStr = strtr($str, $map);
$priceStr = str_replace('万', '', $priceStr);
return (float)trim($priceStr);
}
/**
* 从URL中提取ID
*/
private function extractIdFromUrl(string $url): int
{
if (preg_match('/(\d+)\.html?$/', $url, $matches)) {
return (int)$matches[1];
}
if (preg_match('/\/usedcar\/(\d+)/', $url, $matches)) {
return (int)$matches[1];
}
return 0;
}
/**
* 从参数列表中获取指定名称的值
*/
private function getParamValue(array $params, string $name): ?string
{
foreach ($params as $item) {
if (($item['name'] ?? null) === $name) {
return $item['value'] ?? null;
}
}
return null;
}
/**
* 将GB2312编码转换为UTF-8
*/
private function convertToUtf8(string $html): string
{
if (!function_exists('mb_convert_encoding')) {
return $html;
}
$isGb = stripos($html, 'charset=gb2312') !== false || stripos($html, 'charset=gbk') !== false;
if (!$isGb) {
return $html;
}
$out = @mb_convert_encoding($html, 'UTF-8', 'GB2312');
if ($out === false || $out === '') {
$out = @mb_convert_encoding($html, 'UTF-8', 'GBK');
}
if ($out !== false && $out !== '') {
$html = preg_replace('/charset\s*=\s*["\']?gb(2312|k)["\']?/i', 'charset="utf-8"', $html);
return $out;
}
return $html;
}
/**
* 下载图片到本地
*
* @param string $url 图片URL
* @param string $saveDir 保存目录
* @return string|null 保存的文件路径,失败返回null
*/
public function downloadImage(string $url, string $saveDir): ?string
{
try {
if (!is_dir($saveDir)) {
mkdir($saveDir, 0755, true);
}
$filename = md5($url) . '.jpg';
$savePath = rtrim($saveDir, '/') . '/' . $filename;
// 如果文件已存在,直接返回
if (file_exists($savePath)) {
return $savePath;
}
// 使用 curl 下载图片
$ch = curl_init($url);
curl_setopt_array($ch, [
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_CONNECTTIMEOUT => 5,
CURLOPT_TIMEOUT => 15,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_SSL_VERIFYHOST => false,
CURLOPT_HTTPHEADER => [
'User-Agent: ' . self::DEFAULT_HEADERS['User-Agent'],
'Referer: ' . self::DEFAULT_HEADERS['Referer'],
],
]);
$data = curl_exec($ch);
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
curl_close($ch);
if ($httpCode !== 200 || $data === false) {
return null;
}
file_put_contents($savePath, $data);
return $savePath;
} catch (\Exception $e) {
error_log('下载图片失败: ' . $e->getMessage());
return null;
}
}
/**
* 批量下载图片
*
* @param array $urls 图片URL数组
* @param string $saveDir 保存目录
* @return array [url => localPath]
*/
public function downloadImages(array $urls, string $saveDir): array
{
$result = [];
foreach ($urls as $url) {
$url = trim($url);
if (empty($url)) {
continue;
}
// 处理协议相对URL
if (str_starts_with($url, '//')) {
$url = 'https:' . $url;
}
$localPath = $this->downloadImage($url, $saveDir);
$result[$url] = $localPath;
}
return $result;
}
/**
* 通过关键词搜索懂车帝车系信息
*
* @param string $keyword 搜索关键词(如 "奥迪A4")
* @return array 品牌和车系信息
* @throws \Exception
*/
public function searchByKeyword(string $keyword): array
{
// 访问懂车帝搜索页面获取第一个结果的链接
$searchUrl = "https://www.dongchedi.com/search?keyword=" . urlencode($keyword);
try {
$html = $this->fetchHtml($searchUrl);
$ql = QueryList::html($html);
// 查找"查看全部"按钮的链接
$href = '';
$ql->find('a.fold-btn')->each(function ($item) use (&$href) {
$itemHref = $item->attr('href');
if ($itemHref) {
$href = $itemHref;
}
});
if (empty($href)) {
throw new \Exception('未找到相关车系,请使用更精确的关键词');
}
// 提取车系ID
$seriesId = $this->extractSeriesId($href);
if (!$seriesId) {
throw new \Exception('无法解析车系ID');
}
// 获取品牌和车系信息
return $this->getBrandSeries($seriesId);
} catch (\Exception $e) {
throw new \Exception('搜索失败: ' . $e->getMessage());
}
}
/**
* 获取品牌和车系名称
*
* @param string $seriesId 车系ID
* @return array 包含品牌和车系信息的数组
* @throws \Exception
*/
public function getBrandSeries(string $seriesId): array
{
$url = "https://www.dongchedi.com/auto/series/" . $seriesId . "?anchorId=carModels";
try {
$html = $this->fetchHtml($url);
$ql = QueryList::html($html);
// 从面包屑导航中提取品牌和车系
$breadcrumbs = [];
$ql->find('div.breadcrumb_wrap__2GLeU span')->each(function ($item) use (&$breadcrumbs) {
$breadcrumbs[] = trim($item->text());
});
$brand = $breadcrumbs[2] ?? '';
$series = $breadcrumbs[3] ?? '';
// 获取车型列表
$models = $this->getSeriesModels($seriesId);
return [
'brand' => $brand,
'series' => $series,
'series_id' => $seriesId,
'models' => $models,
];
} catch (\Exception $e) {
throw new \Exception('获取车系信息失败: ' . $e->getMessage());
}
}
/**
* 获取车系下的车型列表
*
* @param string $seriesId 车系ID
* @return array 车型列表
*/
public function getSeriesModels(string $seriesId): array
{
$url = "https://www.dongchedi.com/auto/series/" . $seriesId . "?anchorId=carModels";
$models = [];
try {
$html = $this->fetchHtml($url);
$ql = QueryList::html($html);
// 使用 Tailwind CSS 类名选择器(懂车帝使用 Tailwind)
$nodes = $ql->find('div.tw-grid div.tw-relative div.tw-items-center>a');
$texts = $nodes->texts()->all();
$links = $nodes->attrs('href')->all();
// 提取价格信息(可选)
$priceNodes = $ql->find('div.tw-grid div.tw-text-color-gray-800');
$prices = $priceNodes->texts()->all();
foreach ($texts as $index => $text) {
$modelName = trim($text);
if (empty($modelName)) {
continue;
}
$link = $links[$index] ?? '';
if (!empty($link) && strpos($link, 'http') !== 0) {
$link = 'https://www.dongchedi.com' . $link;
}
$priceStr = $prices[$index] ?? '';
$msrp = 0;
if (!empty($priceStr)) {
$msrp = (int)(str_replace("万", "", $priceStr) * 10000);
}
$models[] = [
'id' => $index,
'name' => $modelName,
'url' => $link,
'msrp' => $msrp,
];
}
return $models;
} catch (\Exception $e) {
// 车型解析失败,记录日志并返回空数组
error_log('获取车型列表失败: ' . $e->getMessage());
return [];
}
}
/**
* 从懂车帝URL中提取车系ID
*
* @param string $url 懂车帝URL
* @return string|null 车系ID
*/
private function extractSeriesId(string $url): ?string
{
// 匹配 /auto/series/x123/ 格式
if (preg_match('/\/auto\/series\/(\d+)/', $url, $matches)) {
return $matches[1];
}
// 匹配 /auto/series/x123/params/ 格式
if (preg_match('/\/auto\/series\/(\d+)/', $url, $matches)) {
return $matches[1];
}
return null;
}
}