xinyouhe/useds
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch '20260116'

Browse Source
This commit is contained in:
WIN-C27C5MSVTPO\16961
2026-01-23 14:18:29 +08:00
parent d949fdbd02 68a27d645c
commit e98b29030b
4 changed files with 363 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/common/utils/crawl/service/UsedCarCrawlSingle.php
+288 -7
View File
@@ -36,10 +36,55 @@ class UsedCarCrawlSingle
'headers' => self::DEFAULT_HEADERS,
'timeout' => 10,
]);
return (string)$response->getBody();
}
/**
* 获取懂车帝移动端分享短链最终url
*
* @param string $url
* @return string
* @throws GuzzleException
*/
public function getFinaUrl(string $url): string
{
$client = new Client([
'verify' => false,
]);
$response = $client->get($url, [
'allow_redirects' => ['track_redirects' => true],
]);
$history = $response->getHeader('X-Guzzle-Redirect-History');
$finalUrl = end($history);
$pcUrl = $this->getPcUrl($finalUrl);
return $pcUrl;
}
/**
* 从移动端url获取sku_id
* @param string $url
* @return string|null
*/
public function getPcUrl(string $url): ?string
{
$query = parse_url($url, PHP_URL_QUERY);
if (!$query) {
return null;
}
parse_str($query, $params);
return !empty($params['sku_id']) ? 'https://www.dongchedi.com/usedcar/'.$params['sku_id'] : null;
}
/**
* 抓取懂车帝二手车信息
* @param string $url
* @return array
* @throws GuzzleException
*/
public function crawlCar(string $url){
$html = $this->fetchHtml($url);
if ($html === '' || str_contains($html, '404 Not Found')) {
@@ -68,20 +113,206 @@ class UsedCarCrawlSingle
return $info;
}
/**
* 通过汽车之家分享链接获取pc端地址
*
* @param string $url
* @return string
* @throws GuzzleException
*/
public function get168FinaUrl(string $url): string
{
$client = new Client([
'verify' => false,
'timeout' => 10,
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Linux; Android 13; Mobile)'
]
]);
$response = $client->get($url);
$html = (string) $response->getBody();
preg_match(
'/var\s+oUrl\s*=\s*[\'"]([^\'"]+che168\.com[^\'"]+)[\'"]/',
$html,
$m
);
if(empty($m[1])){ //如果没有获取到移动端地址
throw new \Exception('168移动端地址解析失败');
}
$pcUrl = $this->getChe168UrlByMobile($m[1]);
return $pcUrl;
}
/**
* 根据移动端url获取汽车之家pc端url
*
* @param string $url
* @return string
* @throws GuzzleException
*/
public function getChe168UrlByMobile(string $url): string
{
// 解析 URL获取车辆id
$parsedUrl = parse_url($url);
$path = $parsedUrl['path']; // "/dealer/162138/56723550.html"
// 使用 pathinfo 获取文件名
$pathParts = explode('/', $path);
$filename = end($pathParts); // "56723550.html"
$carId = pathinfo($filename, PATHINFO_FILENAME); // "56723550"
$pcUrl = 'https://www.che168.com/dealer/111/' . $carId . '.html?pvareaid=0';
return $pcUrl;
}
/**
* 根据汽车之家pc端url转换为另一种页面的汽车之家pc端url
*
* @param string $url
* @return string
* @throws GuzzleException
*/
public function getChe168UrlByPc(string $url): string
{
// 解析URL获取查询参数部分
$queryString = parse_url($url, PHP_URL_QUERY);
// 将查询参数解析为数组
parse_str($queryString, $params);
// 获取infoid
$carId = $params['infoid'] ?? null;
$pcUrl = 'https://www.che168.com/dealer/111/' . $carId . '.html?pvareaid=0';
return $pcUrl;
}
/**
* 抓取汽车之家数据
* @param Request $request
*/
public function crawlChe168Car(string $url)
{
// 解析 URL获取车辆id
$parsedUrl = parse_url($url);
$path = $parsedUrl['path']; // "/dealer/162138/56723550.html"
// 使用 pathinfo 获取文件名
$pathParts = explode('/', $path);
$filename = end($pathParts); // "56723550.html"
$carId = pathinfo($filename, PATHINFO_FILENAME); // "56723550"
// 初始化 Guzzle 客户端
$client = new Client([
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Referer' => 'https://www.che168.com/',
],
'verify' => false,
'timeout' => 10,
]);
// 请求 HTML
$response = $client->get($url);
$html = (string)$response->getBody();
file_put_contents(runtime_path('craw.html'), $html);
// 检查是否获取到 HTML
if (empty($html)) {
throw new \Exception('无法获取页面 HTML');
}
// che168 页面为 gb2312，Linux 默认 UTF-8，直接解析会导致 DOM 解析异常。统一转 UTF-8 再解析。
$html = $this->ensureUtf8ForChe168($html);
// 使用 QueryList 解析 HTML
$ql = QueryList::html($html);
// 解析车辆列表，每辆车可能在 li.cards-li 或类似类名里
$cars = $ql->find('div#nav1 ul.basic-item-ul li')->map(function($item) {
$attribute = [];
$field = $item->find('.item-name')->text();
$value = $item->text();
$title = ['product_name' => '产品名称','plate_time' => '上牌时间', 'mileage' => '表显里程', 'capacity' => '排       量', 'source_place'=> '所  在  地', 'body_color'=>'车身颜色','condition_desc'=>'车况介绍',];
$title_key = array_flip($title);
if(in_array($field,$title)){
$attribute[$title_key[$field]] = str_replace($field, '', $value);
}
return $attribute;
})->all();
$cars = array_filter($cars);
$cars = array_merge(...$cars);
$cars['product_name'] = $cars['car_name'] = $ql->find('div.car-box .car-brand-name')->text();
$cars['price'] = $ql->find('div.goodstartmoney')->text();
if(empty($cars['price'])){
$cars['price'] = $ql->find('span#overlayPrice')->text();
}
// $cars['price'] = explode('万', $cars['price'])[0]??0;
$cars['price'] = str_replace(['¥', '￥', '万'], '', $cars['price']);
$cars['price'] = is_numeric($cars['price'])?$cars['price']*10000:$cars['price'];
//$cars['price'] = $ql->find('#overlayPrice')->text();
$cars['title'] = $cars['product_name'];
$cars['condition_desc'] = $ql->find('div.leave-message-box #messageBox')->text();
$cars['usage_mileage'] = str_replace('万公里', '', $cars['mileage']);
//品牌与车系
$links = $ql->find('.bread-crumbs a')->map(function ($a) {
return trim($a->text());
})->all();
$cars['brand_name'] = !empty($links[3]) ? str_replace('二手','',$links[3]): null;
$cars['brand_id'] = 0;
$cars['series_name'] = !empty($links[4]) ?str_replace('二手','',$links[4]): null;
$cars['series_id'] = 0;
$cars['car_id'] = 0;
$info = explode(' ',$cars['product_name']);
$info = array_filter($info, function($item) {
return str_contains($item, '款');
});
$year = end($info);
$cars['year'] = !empty($year) ? str_replace('款', '', $year):'';
// 图片：主用轮播 .swiper-container2，备用 #pic_li（懒加载 data-original）
$images = $this->collectChe168Images($ql, '.swiper-container2 .swiper-slide .jiaodianphotoclick img', false);
if (empty($images)) {
$images = $this->collectChe168Images($ql, '#pic_li img', true);
}
$images = array_values(array_unique($images));
var_dump($images);
if (empty($images)) {
Log::warning("crawlChe168Car:未找到任何车辆图片，URL: {$url}");
}
$saveDir = public_path('uploads/usedcar/' . $carId);
$cars['used_car_id'] = $carId;
$image = $this->uploadAndRemove($images, $saveDir, 'https://www.che168.com/');
$cars['image'] = $image['image'];
$cars['image_pull'] = $image['image_pull'];
return $cars;
}
/**
* 下载 -> 上传 -> 删除本地
*
* @param array $images 原始相册URL列表
* @param string $path 本地保存目录
*/
public function uploadAndRemove(array $images, string $path)
public function uploadAndRemove(array $images, string $path, string $referer = '')
{
if (empty($images)) {
return false;
}
// 下载图片
$filePatch = $this->downloadImageAlbum($images, $path);
$filePatch = $this->downloadImageAlbum($images, $path,$referer);
if (empty($filePatch)) {
return false;
}
@@ -144,7 +375,7 @@ class UsedCarCrawlSingle
* @param string $saveDir 保存目录
* @return array [原始URL => 保存路径或null]
*/
public function downloadImageAlbum(array $urls, string $saveDir): array
public function downloadImageAlbum(array $urls, string $saveDir, string $referer): array
{
$result = [];
@@ -159,7 +390,7 @@ class UsedCarCrawlSingle
}
try {
$path = $this->downloadImage($url, $saveDir);
$path = $this->downloadImage($url, $saveDir, $referer);
} catch (Throwable $e) {
$path = null;
}
@@ -174,7 +405,7 @@ class UsedCarCrawlSingle
/**
* 下载单张图片
*/
public function downloadImage(string $url, string $saveDir, ?string $filename = null, array $headers = []): ?string
public function downloadImage(string $url, string $saveDir, string $referer, ?string $filename = null, array $headers = []): ?string
{
if (!is_dir($saveDir)) {
if (!@mkdir($saveDir, 0777, true) && !is_dir($saveDir)) {
@@ -196,10 +427,12 @@ class UsedCarCrawlSingle
$savePath = rtrim($saveDir, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . $filename;
$referer = empty($referer) ? self::DEFAULT_HEADERS['Referer'] : $referer;
$defaultHeaders = [
'User-Agent: ' . self::DEFAULT_HEADERS['User-Agent'],
'Accept: image/avif,image/webp,image/apng,image/*,*/*;q=0.8',
'Referer: ' . self::DEFAULT_HEADERS['Referer'],
// 'Referer: ' . self::DEFAULT_HEADERS['Referer'],
'Referer: ' . $referer,
];
foreach ($headers as $h) {
@@ -416,6 +649,54 @@ class UsedCarCrawlSingle
return null;
}
/**
* 从 che168 页面上收集图片 URL
*
* @param \QL\QueryList $ql
* @param string $selector 选择器
* @param bool $preferDataOriginal 为 true 时优先用 data-original（#pic_li 懒加载）
* @return array<string>
*/
protected function collectChe168Images($ql, string $selector, bool $preferDataOriginal): array
{
return $ql->find($selector)->map(function ($img) use ($preferDataOriginal) {
$src = $preferDataOriginal ? ($img->attr('data-original') ?: $img->attr('src')) : ($img->attr('src') ?: $img->attr('data-original'));
if (!$src || $src === '' || str_contains($src, 'default-che168')) {
return null;
}
if (str_starts_with($src, '//')) {
$src = 'https:' . $src;
}
return $src;
})->filter()->values()->all();
}
/**
* 将 che168 页面 (gb2312/gbk) 转为 UTF-8，避免 Linux 下 DOM 解析异常
*
* @param string $html
* @return string
*/
protected function ensureUtf8ForChe168(string $html): string
{
if (!function_exists('mb_convert_encoding')) {
return $html;
}
$isGb = stripos($html, 'charset=gb2312') !== false || stripos($html, 'charset=gbk') !== false;
if (!$isGb) {
return $html;
}
$out = @mb_convert_encoding($html, 'UTF-8', 'GB2312');
if ($out === false || $out === '') {
$out = @mb_convert_encoding($html, 'UTF-8', 'GBK');
}
if ($out !== false && $out !== '') {
$html = $out;
$html = preg_replace('/charset\s*=\s*["\']?gb(2312|k)["\']?/i', 'charset="utf-8"', $html);
}
return $html;
}
/**
* 安全解 JSON
*