From 650f9430374d82a4f322d5e827b3bcd30648f979 Mon Sep 17 00:00:00 2001 From: "WIN-C27C5MSVTPO\\16961" <1696136552@qq.com> Date: Wed, 21 Jan 2026 14:12:59 +0800 Subject: [PATCH 01/10] =?UTF-8?q?=E7=A7=BB=E5=8A=A8=E7=AB=AFurl=E6=8A=93?= =?UTF-8?q?=E5=8F=96=E6=87=82=E8=BD=A6=E5=B8=9D=E8=BD=A6=E8=BE=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/common/controller/UsedCarController.php | 19 +++++++++ .../crawl/service/UsedCarCrawlSingle.php | 42 ++++++++++++++++++- config/route.php | 3 +- 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/app/common/controller/UsedCarController.php b/app/common/controller/UsedCarController.php index b2f2a94..2b1365b 100644 --- a/app/common/controller/UsedCarController.php +++ b/app/common/controller/UsedCarController.php @@ -128,6 +128,25 @@ class UsedCarController extends BaseController return $data; } + /** + * 移动端链接抓取 + * @param Request $request + */ + public function crawlMobileCar(Request $request) + { + try { + $urlShare = $request->get('url'); + + $finalUrl = (new UsedCarCrawlSingle())->getFinaUrl($urlShare); + + $request->setGet('url', $finalUrl); + $data = $this->crawlCar($request); + return $data; + } catch (\Throwable $e) { + return $this->error($e->getMessage()); + } + } + public function crawlCar(Request $request) { try { diff --git a/app/common/utils/crawl/service/UsedCarCrawlSingle.php b/app/common/utils/crawl/service/UsedCarCrawlSingle.php index c04cd10..0d6bf26 100644 --- a/app/common/utils/crawl/service/UsedCarCrawlSingle.php +++ b/app/common/utils/crawl/service/UsedCarCrawlSingle.php @@ -36,10 +36,50 @@ class UsedCarCrawlSingle 'headers' => self::DEFAULT_HEADERS, 'timeout' => 10, ]); - return (string)$response->getBody(); } + /** + * 获取移动端最终url + * + * @param string $url + * @return string + * @throws GuzzleException + */ + public function getFinaUrl(string $url): string + { + $client = new Client([ + 'verify' => false, + ]); + $response = $client->get($url, [ + 'allow_redirects' => ['track_redirects' => true], + ]); + + $history = $response->getHeader('X-Guzzle-Redirect-History'); + $finalUrl = end($history); + + $sku_id = $this->getSkuIdFromUrl($finalUrl); + + return 'https://www.dongchedi.com/usedcar/'.$sku_id; + } + + /** + * 从url获取sku_id + * @param string $url + * @return string|null + */ + public function getSkuIdFromUrl(string $url): ?string + { + $query = parse_url($url, PHP_URL_QUERY); + if (!$query) { + return null; + } + + parse_str($query, $params); + + return $params['sku_id'] ?? null; + } + public function crawlCar(string $url){ $html = $this->fetchHtml($url); if ($html === '' || str_contains($html, '404 Not Found')) { diff --git a/config/route.php b/config/route.php index f6b0373..b9337f4 100644 --- a/config/route.php +++ b/config/route.php @@ -68,12 +68,11 @@ Route::group('/common', function () { Route::any('/destroy', [app\common\controller\UsedCarController::class, 'destroy']); Route::any('/crawl', [app\common\controller\UsedCarController::class, 'crawlKeyword']); Route::any('/url_crawl', [app\common\controller\UsedCarController::class, 'crawlCar']); + Route::any('/mobile/url_crawl', [app\common\controller\UsedCarController::class, 'crawlMobileCar']); }); Route::group('/user', function () { Route::any('/info', [app\common\controller\UserController::class, 'show']); Route::any('/edit', [app\common\controller\UserController::class, 'edit']); }); - - }); \ No newline at end of file From 892325008c8829ba65d8d38bd7e1b04067cff978 Mon Sep 17 00:00:00 2001 From: "WIN-C27C5MSVTPO\\16961" <1696136552@qq.com> Date: Wed, 21 Jan 2026 17:03:42 +0800 Subject: [PATCH 02/10] =?UTF-8?q?=E6=87=82=E8=BD=A6=E5=B8=9D=E7=A7=BB?= =?UTF-8?q?=E5=8A=A8=E7=AB=AF=E6=8A=93=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/common/controller/UsedCarController.php | 3 +- app/common/service/UsedCarService.php | 35 +++++++++++++++++++ .../crawl/service/UsedCarCrawlSingle.php | 18 ++++++---- 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/app/common/controller/UsedCarController.php b/app/common/controller/UsedCarController.php index 2b1365b..d5c4ee6 100644 --- a/app/common/controller/UsedCarController.php +++ b/app/common/controller/UsedCarController.php @@ -154,7 +154,8 @@ class UsedCarController extends BaseController if (empty($url)){ throw new BusinessException('url不能为空'); } - $info = (new UsedCarCrawlSingle())->crawlCar($url); + //$info = (new UsedCarCrawlSingle())->crawlCar($url); + $info = $this->service->selectCraw($url); $data = [ 'user_id' => $request->user['id'], 'product_name' => $info['title'], diff --git a/app/common/service/UsedCarService.php b/app/common/service/UsedCarService.php index a3692ec..7e29b9b 100644 --- a/app/common/service/UsedCarService.php +++ b/app/common/service/UsedCarService.php @@ -2,6 +2,7 @@ namespace app\common\service; +use app\common\utils\crawl\service\UsedCarCrawlSingle; use app\common\validate\UsedCarValidate; use app\common\traits\crud\CrudServiceTrait; use app\common\model\UsedCar; @@ -421,4 +422,38 @@ class UsedCarService return $path; } + + /** + * 选择车辆抓取方式 + * @param string $url + */ + public function selectCraw(string $url){ + $usedCarCrawl = new UsedCarCrawlSingle(); + switch (true){ + case str_contains($url,'dongchedi.com'): //懂车帝pc端 + $info = $usedCarCrawl->crawlCar($url); + break; + case str_contains($url,'dcdapp.com'): //懂车帝移动端 + $finalUrl = $usedCarCrawl->getPcUrl($url); //获取pc端页面链接 + if(empty($finalUrl)){ + throw new \Exception('链接解析错误'); + } + $info = $usedCarCrawl->crawlCar($finalUrl); + break; + case str_contains($url,'t.zijieimg.com'): //懂车帝移动端分享链接 + $finalUrl = $usedCarCrawl->getFinaUrl($url); //解析抖音短链,并获取pc端页面链接 + if(empty($finalUrl)){ + throw new \Exception('链接解析错误'); + } + $info = $usedCarCrawl->crawlCar($finalUrl); + break; + case str_contains($url,'che168.com'): //汽车之家 + throw new \Exception('加紧开发中'); + break; + default: + Log::info('未知链接:'.$url); + throw new \Exception('加紧开发中'); + } + return $info; + } } diff --git a/app/common/utils/crawl/service/UsedCarCrawlSingle.php b/app/common/utils/crawl/service/UsedCarCrawlSingle.php index 0d6bf26..1f45f1b 100644 --- a/app/common/utils/crawl/service/UsedCarCrawlSingle.php +++ b/app/common/utils/crawl/service/UsedCarCrawlSingle.php @@ -40,7 +40,7 @@ class UsedCarCrawlSingle } /** - * 获取移动端最终url + * 获取懂车帝移动端分享短链最终url * * @param string $url * @return string @@ -58,17 +58,17 @@ class UsedCarCrawlSingle $history = $response->getHeader('X-Guzzle-Redirect-History'); $finalUrl = end($history); - $sku_id = $this->getSkuIdFromUrl($finalUrl); + $pcUrl = $this->getPcUrl($finalUrl); - return 'https://www.dongchedi.com/usedcar/'.$sku_id; + return $pcUrl; } /** - * 从url获取sku_id + * 从移动端url获取sku_id * @param string $url * @return string|null */ - public function getSkuIdFromUrl(string $url): ?string + public function getPcUrl(string $url): ?string { $query = parse_url($url, PHP_URL_QUERY); if (!$query) { @@ -77,9 +77,15 @@ class UsedCarCrawlSingle parse_str($query, $params); - return $params['sku_id'] ?? null; + return !empty($params['sku_id']) ? 'https://www.dongchedi.com/usedcar/'.$params['sku_id'] : null; } + /** + * 抓取懂车帝二手车信息 + * @param string $url + * @return array + * @throws GuzzleException + */ public function crawlCar(string $url){ $html = $this->fetchHtml($url); if ($html === '' || str_contains($html, '404 Not Found')) { From c7d3fdc30ebb0538d911875c245504550edfb619 Mon Sep 17 00:00:00 2001 From: "WIN-C27C5MSVTPO\\16961" <1696136552@qq.com> Date: Thu, 22 Jan 2026 13:29:01 +0800 Subject: [PATCH 03/10] =?UTF-8?q?=E6=B1=BD=E8=BD=A6=E4=B9=8B=E5=AE=B6?= =?UTF-8?q?=E6=8A=93=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/common/controller/UsedCarController.php | 4 +- app/common/service/UsedCarService.php | 4 +- .../crawl/service/UsedCarCrawlSingle.php | 108 ++++++++++++++++++ 3 files changed, 112 insertions(+), 4 deletions(-) diff --git a/app/common/controller/UsedCarController.php b/app/common/controller/UsedCarController.php index d5c4ee6..5eba0d4 100644 --- a/app/common/controller/UsedCarController.php +++ b/app/common/controller/UsedCarController.php @@ -161,9 +161,9 @@ class UsedCarController extends BaseController 'product_name' => $info['title'], 'price' => $info['price']<200 ? $info['price']*10000 : $info['price'], 'source_place' => $info['source_place'], - 'displacement' => $info['capacity'], //排量 + 'displacement' => $info['capacity']??'', //排量 'mileage' => $info['usage_mileage']*10000, //里程 - 'power' => $info['horsepower'], //马力 + 'power' => $info['horsepower']??'', //马力 'body_color' => $info['body_color'], 'images' => $info['image'], 'image_pull' => $info['image_pull'], diff --git a/app/common/service/UsedCarService.php b/app/common/service/UsedCarService.php index 7e29b9b..fc6fcc1 100644 --- a/app/common/service/UsedCarService.php +++ b/app/common/service/UsedCarService.php @@ -447,8 +447,8 @@ class UsedCarService } $info = $usedCarCrawl->crawlCar($finalUrl); break; - case str_contains($url,'che168.com'): //汽车之家 - throw new \Exception('加紧开发中'); + case str_contains($url,'che168.com/dealer'): //汽车之家 + $info = $usedCarCrawl->crawlChe168Car($url); break; default: Log::info('未知链接:'.$url); diff --git a/app/common/utils/crawl/service/UsedCarCrawlSingle.php b/app/common/utils/crawl/service/UsedCarCrawlSingle.php index 1f45f1b..c793e60 100644 --- a/app/common/utils/crawl/service/UsedCarCrawlSingle.php +++ b/app/common/utils/crawl/service/UsedCarCrawlSingle.php @@ -114,6 +114,114 @@ class UsedCarCrawlSingle return $info; } + /** + * 抓取汽车之家数据 + * @param Request $request + */ + public function crawlChe168Car(string $url) + { + // 解析 URL获取车辆id + $parsedUrl = parse_url($url); + $path = $parsedUrl['path']; // "/dealer/162138/56723550.html" + // 使用 pathinfo 获取文件名 + $pathParts = explode('/', $path); + $filename = end($pathParts); // "56723550.html" + $carId = pathinfo($filename, PATHINFO_FILENAME); // "56723550" + + // 初始化 Guzzle 客户端 + $client = new Client([ + 'headers' => [ + 'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)', + 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Referer' => 'https://www.che168.com/', + ], + 'verify' => false, + 'timeout' => 10, + ]); + + + // 请求 HTML + $response = $client->get($url); + $html = (string)$response->getBody(); + + // 检查是否获取到 HTML + if (empty($html)) { + throw new \Exception('无法获取页面 HTML'); + } + + // 使用 QueryList 解析 HTML + $ql = QueryList::html($html); + file_put_contents(runtime_path('168.html'),$html); + + // 解析车辆列表,每辆车可能在 li.cards-li 或类似类名里 + $cars = $ql->find('div#nav1 ul.basic-item-ul li')->map(function($item) { + $attribute = []; + $field = $item->find('.item-name')->text(); + $value = $item->text(); + $title = ['product_name' => '产品名称','plate_time' => '上牌时间', 'mileage' => '表显里程', 'capacity' => '排       量', 'source_place'=> '所  在  地', 'body_color'=>'车身颜色','condition_desc'=>'车况介绍',]; + $title_key = array_flip($title); + if(in_array($field,$title)){ + $attribute[$title_key[$field]] = str_replace($field, '', $value); + } + + // 车辆图片 +// $img = $item->find('.list-photo img')->attr('src'); + + return $attribute; + })->all(); + + $cars = array_filter($cars); + $cars = array_merge(...$cars); + $cars['product_name'] = $cars['car_name'] = $ql->find('div.car-box .car-brand-name')->text(); + $cars['price'] = $ql->find('span.price')->text(); + $cars['price'] = !empty($cars['price'])?str_replace('万', '', $cars['price'])*10000:0; + //$cars['price'] = $ql->find('#overlayPrice')->text(); + $cars['title'] = $cars['product_name']; + $cars['condition_desc'] = $ql->find('div.leave-message-box #messageBox')->text(); + $cars['usage_mileage'] = str_replace('万公里', '', $cars['mileage']); + + //品牌与车系 + $links = $ql->find('.bread-crumbs a')->map(function ($a) { + return trim($a->text()); + })->all(); + $cars['brand_name'] = $links[3] ?str_replace('二手','',$links[3]): null; + $cars['brand_id'] = 0; + $cars['series_name'] = $links[4] ?str_replace('二手','',$links[4]): null; + $cars['series_id'] = 0; + $cars['car_id'] = 0; + $info = explode(' ',$cars['product_name']); + $info = array_filter($info, function($item) { + return str_contains($item, '款'); + }); + $year = end($info); + $cars['year'] = !empty($year) ? str_replace('款', '', $year):''; + + //图片 + $images = $ql->find('#pic_li img')->map(function ($img) { + // 只取 data-original + $src = $img->attr('data-original'); + + if (!$src) { + return null; + } + + // 补全 https + if (str_starts_with($src, '//')) { + $src = 'https:' . $src; + } + + return $src; + })->filter()->values()->all(); + + $saveDir = public_path('uploads/usedcar/' . $carId); + $cars['used_car_id'] = $carId; + $image = $this->uploadAndRemove($images, $saveDir); + $cars['image'] = $image['image']; + $cars['image_pull'] = $image['image_pull']; + + return $cars; + } + /** * 下载 -> 上传 -> 删除本地 * From da1dc7a3dad3da7b1d8921337894c0ce4e327cd0 Mon Sep 17 00:00:00 2001 From: "WIN-C27C5MSVTPO\\16961" <1696136552@qq.com> Date: Thu, 22 Jan 2026 18:08:51 +0800 Subject: [PATCH 04/10] =?UTF-8?q?=E6=B1=BD=E8=BD=A6=E4=B9=8B=E5=AE=B6?= =?UTF-8?q?=E6=8A=93=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/common/controller/UsedCarController.php | 4 +- app/common/service/UsedCarService.php | 14 +++- .../crawl/service/UsedCarCrawlSingle.php | 65 +++++++++++++++++-- 3 files changed, 77 insertions(+), 6 deletions(-) diff --git a/app/common/controller/UsedCarController.php b/app/common/controller/UsedCarController.php index 5eba0d4..c51c5a5 100644 --- a/app/common/controller/UsedCarController.php +++ b/app/common/controller/UsedCarController.php @@ -11,6 +11,7 @@ use app\common\service\UsedCarService; use app\common\utils\crawl\service\UsedCarCrawlSingle; use app\common\utils\pdf\service\MpdfDriver; use support\exception\BusinessException; +use support\Log; use support\Request; class UsedCarController extends BaseController @@ -186,7 +187,8 @@ class UsedCarController extends BaseController if (str_contains($e->getMessage(), '404 Not Found')){ return $this->error('车辆已下架或找不到页面'); } - return $this->error($e->getMessage()); + Log::error('crawlCar fail',['param' => $request->get('url'),'message' => $e->getMessage().$e->getFile().$e->getLine()]); + return $this->error($e->getMessage().$e->getFile().$e->getLine()); } } diff --git a/app/common/service/UsedCarService.php b/app/common/service/UsedCarService.php index fc6fcc1..bb71b50 100644 --- a/app/common/service/UsedCarService.php +++ b/app/common/service/UsedCarService.php @@ -447,7 +447,19 @@ class UsedCarService } $info = $usedCarCrawl->crawlCar($finalUrl); break; - case str_contains($url,'che168.com/dealer'): //汽车之家 + case str_contains($url,'www.che168.com/dealer'): //汽车之家 + $info = $usedCarCrawl->crawlChe168Car($url); + break; + case str_contains($url,'wap.che168.com/dealer'): //汽车之家移动端链接 + $finalUrl = $usedCarCrawl->getChe168UrlByMobile($url); + $info = $usedCarCrawl->crawlChe168Car($finalUrl); + break; + case str_contains($url,'comm.app.autohome.com.cn'): //汽车之家移动端链接 + $finalUrl = $usedCarCrawl->get168FinaUrl($url); + var_dump($finalUrl); + $info = $usedCarCrawl->crawlChe168Car($finalUrl); + break; + case str_contains($url,'www.che168.com/dealer'): //汽车之家 $info = $usedCarCrawl->crawlChe168Car($url); break; default: diff --git a/app/common/utils/crawl/service/UsedCarCrawlSingle.php b/app/common/utils/crawl/service/UsedCarCrawlSingle.php index c793e60..f0e505b 100644 --- a/app/common/utils/crawl/service/UsedCarCrawlSingle.php +++ b/app/common/utils/crawl/service/UsedCarCrawlSingle.php @@ -59,7 +59,62 @@ class UsedCarCrawlSingle $finalUrl = end($history); $pcUrl = $this->getPcUrl($finalUrl); + return $pcUrl; + } + /** + * 通过汽车之家分享链接获取pc端地址 + * + * @param string $url + * @return string + * @throws GuzzleException + */ + public function get168FinaUrl(string $url): string + { + $client = new Client([ + 'verify' => false, + 'timeout' => 10, + 'headers' => [ + 'User-Agent' => 'Mozilla/5.0 (Linux; Android 13; Mobile)' + ] + ]); + + $response = $client->get($url); + $html = (string) $response->getBody(); + + preg_match( + '/var\s+oUrl\s*=\s*[\'"]([^\'"]+che168\.com[^\'"]+)[\'"]/', + $html, + $m + ); + + if(empty($m[1])){ //如果没有获取到移动端地址 + throw new \Exception('168移动端地址解析失败'); + } + + $pcUrl = $this->getChe168UrlByMobile($m[1]); + return $pcUrl; + } + + /** + * 根据移动端url获取汽车之家pc端url + * + * @param string $url + * @return string + * @throws GuzzleException + */ + public function getChe168UrlByMobile(string $url): string + { + + // 解析 URL获取车辆id + $parsedUrl = parse_url($url); + $path = $parsedUrl['path']; // "/dealer/162138/56723550.html" + // 使用 pathinfo 获取文件名 + $pathParts = explode('/', $path); + $filename = end($pathParts); // "56723550.html" + $carId = pathinfo($filename, PATHINFO_FILENAME); // "56723550" + + $pcUrl = 'https://www.che168.com/dealer/111/' . $carId . '.html?pvareaid=0'; return $pcUrl; } @@ -143,6 +198,7 @@ class UsedCarCrawlSingle // 请求 HTML $response = $client->get($url); $html = (string)$response->getBody(); + file_put_contents(runtime_path('craw.html'), $html); // 检查是否获取到 HTML if (empty($html)) { @@ -151,7 +207,6 @@ class UsedCarCrawlSingle // 使用 QueryList 解析 HTML $ql = QueryList::html($html); - file_put_contents(runtime_path('168.html'),$html); // 解析车辆列表,每辆车可能在 li.cards-li 或类似类名里 $cars = $ql->find('div#nav1 ul.basic-item-ul li')->map(function($item) { @@ -174,7 +229,9 @@ class UsedCarCrawlSingle $cars = array_merge(...$cars); $cars['product_name'] = $cars['car_name'] = $ql->find('div.car-box .car-brand-name')->text(); $cars['price'] = $ql->find('span.price')->text(); - $cars['price'] = !empty($cars['price'])?str_replace('万', '', $cars['price'])*10000:0; + $cars['price'] = explode('万', $cars['price'])[0]??0; + $cars['price'] = str_replace(['¥', '¥'], '', $cars['price']); + $cars['price'] = $cars['price']*10000; //$cars['price'] = $ql->find('#overlayPrice')->text(); $cars['title'] = $cars['product_name']; $cars['condition_desc'] = $ql->find('div.leave-message-box #messageBox')->text(); @@ -184,9 +241,9 @@ class UsedCarCrawlSingle $links = $ql->find('.bread-crumbs a')->map(function ($a) { return trim($a->text()); })->all(); - $cars['brand_name'] = $links[3] ?str_replace('二手','',$links[3]): null; + $cars['brand_name'] = !empty($links[3]) ? str_replace('二手','',$links[3]): null; $cars['brand_id'] = 0; - $cars['series_name'] = $links[4] ?str_replace('二手','',$links[4]): null; + $cars['series_name'] = !empty($links[4]) ?str_replace('二手','',$links[4]): null; $cars['series_id'] = 0; $cars['car_id'] = 0; $info = explode(' ',$cars['product_name']); From 323dbac22166f0a35650c68b1bd55aef35b5691b Mon Sep 17 00:00:00 2001 From: "WIN-C27C5MSVTPO\\16961" <1696136552@qq.com> Date: Fri, 23 Jan 2026 10:38:36 +0800 Subject: [PATCH 05/10] =?UTF-8?q?=E6=B1=BD=E8=BD=A6=E4=B9=8B=E5=AE=B6?= =?UTF-8?q?=E9=93=BE=E6=8E=A5=E8=BD=AC=E6=8D=A2=EF=BC=8C=E6=8A=93=E5=8F=96?= =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/common/service/UsedCarService.php | 9 ++-- .../crawl/service/UsedCarCrawlSingle.php | 54 +++++++++++++++---- 2 files changed, 48 insertions(+), 15 deletions(-) diff --git a/app/common/service/UsedCarService.php b/app/common/service/UsedCarService.php index bb71b50..31ef9eb 100644 --- a/app/common/service/UsedCarService.php +++ b/app/common/service/UsedCarService.php @@ -7,6 +7,7 @@ use app\common\validate\UsedCarValidate; use app\common\traits\crud\CrudServiceTrait; use app\common\model\UsedCar; use GuzzleHttp\Client; +use GuzzleHttp\Exception\GuzzleException; use support\exception\BusinessException; use support\Log; use support\Request; @@ -450,18 +451,18 @@ class UsedCarService case str_contains($url,'www.che168.com/dealer'): //汽车之家 $info = $usedCarCrawl->crawlChe168Car($url); break; + case str_contains($url,'www.che168.com/home?infoid='): //汽车之家pc端链接转换 + $url = $usedCarCrawl->getChe168UrlByPc($url); + $info = $usedCarCrawl->crawlChe168Car($url); + break; case str_contains($url,'wap.che168.com/dealer'): //汽车之家移动端链接 $finalUrl = $usedCarCrawl->getChe168UrlByMobile($url); $info = $usedCarCrawl->crawlChe168Car($finalUrl); break; case str_contains($url,'comm.app.autohome.com.cn'): //汽车之家移动端链接 $finalUrl = $usedCarCrawl->get168FinaUrl($url); - var_dump($finalUrl); $info = $usedCarCrawl->crawlChe168Car($finalUrl); break; - case str_contains($url,'www.che168.com/dealer'): //汽车之家 - $info = $usedCarCrawl->crawlChe168Car($url); - break; default: Log::info('未知链接:'.$url); throw new \Exception('加紧开发中'); diff --git a/app/common/utils/crawl/service/UsedCarCrawlSingle.php b/app/common/utils/crawl/service/UsedCarCrawlSingle.php index f0e505b..e3372ad 100644 --- a/app/common/utils/crawl/service/UsedCarCrawlSingle.php +++ b/app/common/utils/crawl/service/UsedCarCrawlSingle.php @@ -118,6 +118,28 @@ class UsedCarCrawlSingle return $pcUrl; } + /** + * 根据汽车之家pc端url转换为另一种页面的汽车之家pc端url + * + * @param string $url + * @return string + * @throws GuzzleException + */ + public function getChe168UrlByPc(string $url): string + { + // 解析URL获取查询参数部分 + $queryString = parse_url($url, PHP_URL_QUERY); + // 将查询参数解析为数组 + parse_str($queryString, $params); + + // 获取infoid + $carId = $params['infoid'] ?? null; + + $pcUrl = 'https://www.che168.com/dealer/111/' . $carId . '.html?pvareaid=0'; + + return $pcUrl; + } + /** * 从移动端url获取sku_id * @param string $url @@ -219,18 +241,18 @@ class UsedCarCrawlSingle $attribute[$title_key[$field]] = str_replace($field, '', $value); } - // 车辆图片 -// $img = $item->find('.list-photo img')->attr('src'); - return $attribute; })->all(); $cars = array_filter($cars); $cars = array_merge(...$cars); $cars['product_name'] = $cars['car_name'] = $ql->find('div.car-box .car-brand-name')->text(); - $cars['price'] = $ql->find('span.price')->text(); - $cars['price'] = explode('万', $cars['price'])[0]??0; - $cars['price'] = str_replace(['¥', '¥'], '', $cars['price']); + $cars['price'] = $ql->find('div.goodstartmoney')->text(); + if(empty($cars['price'])){ + $cars['price'] = $ql->find('span#overlayPrice')->text(); + } +// $cars['price'] = explode('万', $cars['price'])[0]??0; + $cars['price'] = str_replace(['¥', '¥', '万'], '', $cars['price']); $cars['price'] = $cars['price']*10000; //$cars['price'] = $ql->find('#overlayPrice')->text(); $cars['title'] = $cars['product_name']; @@ -254,12 +276,13 @@ class UsedCarCrawlSingle $cars['year'] = !empty($year) ? str_replace('款', '', $year):''; //图片 - $images = $ql->find('#pic_li img')->map(function ($img) { - // 只取 data-original - $src = $img->attr('data-original'); + $images = $ql->find('.swiper-container2 .swiper-slide .jiaodianphotoclick img')->map(function ($img) { + // 优先尝试 data-original(懒加载图片) + $src = $img->attr('src'); - if (!$src) { - return null; + // 如果没有 data-original,尝试 src + if (!$src || $src === '' ||str_contains($src, 'default-che168')) { + $src = $img->attr('data-original'); } // 补全 https @@ -267,9 +290,18 @@ class UsedCarCrawlSingle $src = 'https:' . $src; } + // 过滤掉占位符图片 + if (str_contains($src, 'default-che168')) { + return null; + } + return $src; })->filter()->values()->all(); + if (empty($images)) { + Log::warning("crawlChe168Car:未找到任何车辆图片,URL: {$url}"); + } + $saveDir = public_path('uploads/usedcar/' . $carId); $cars['used_car_id'] = $carId; $image = $this->uploadAndRemove($images, $saveDir); From 140e00821cb5ea4ad5193af255588ad4bb10328c Mon Sep 17 00:00:00 2001 From: "WIN-C27C5MSVTPO\\16961" <1696136552@qq.com> Date: Fri, 23 Jan 2026 10:40:03 +0800 Subject: [PATCH 06/10] =?UTF-8?q?=E8=B0=83=E6=95=B4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../crawl/service/UsedCarCrawlSingle.php | 102 +++++++++--------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/app/common/utils/crawl/service/UsedCarCrawlSingle.php b/app/common/utils/crawl/service/UsedCarCrawlSingle.php index e3372ad..4e079ce 100644 --- a/app/common/utils/crawl/service/UsedCarCrawlSingle.php +++ b/app/common/utils/crawl/service/UsedCarCrawlSingle.php @@ -62,6 +62,57 @@ class UsedCarCrawlSingle return $pcUrl; } + /** + * 从移动端url获取sku_id + * @param string $url + * @return string|null + */ + public function getPcUrl(string $url): ?string + { + $query = parse_url($url, PHP_URL_QUERY); + if (!$query) { + return null; + } + + parse_str($query, $params); + + return !empty($params['sku_id']) ? 'https://www.dongchedi.com/usedcar/'.$params['sku_id'] : null; + } + + /** + * 抓取懂车帝二手车信息 + * @param string $url + * @return array + * @throws GuzzleException + */ + public function crawlCar(string $url){ + $html = $this->fetchHtml($url); + if ($html === '' || str_contains($html, '404 Not Found')) { + throw new BusinessException("页面内容为空: {$url}"); + } + + $info = $this->parseUsedCarDetail($html, $url); + +// $insertId = $this->saveUsedCarBasicInfo($info); + + // 下载图片 + 上传 + 入库(图片失败不影响主记录) + $imagesArray = json_decode($info['images'] ?? '[]', true); + if (json_last_error() !== JSON_ERROR_NONE) { + $imagesArray = []; + } + $saveDir = public_path('uploads/usedcar/' . $info['used_car_id']); + $image = $this->uploadAndRemove($imagesArray, $saveDir); + $info['image'] = $image['image']; + $info['image_pull'] = $image['image_pull']; + + // 继续派发配置页任务 +// if (!empty($info['car_id_url'])) { +// $this->pushUrl($info['car_id_url'], 'config', $insertId); +// } + + return $info; + } + /** * 通过汽车之家分享链接获取pc端地址 * @@ -140,57 +191,6 @@ class UsedCarCrawlSingle return $pcUrl; } - /** - * 从移动端url获取sku_id - * @param string $url - * @return string|null - */ - public function getPcUrl(string $url): ?string - { - $query = parse_url($url, PHP_URL_QUERY); - if (!$query) { - return null; - } - - parse_str($query, $params); - - return !empty($params['sku_id']) ? 'https://www.dongchedi.com/usedcar/'.$params['sku_id'] : null; - } - - /** - * 抓取懂车帝二手车信息 - * @param string $url - * @return array - * @throws GuzzleException - */ - public function crawlCar(string $url){ - $html = $this->fetchHtml($url); - if ($html === '' || str_contains($html, '404 Not Found')) { - throw new BusinessException("页面内容为空: {$url}"); - } - - $info = $this->parseUsedCarDetail($html, $url); - -// $insertId = $this->saveUsedCarBasicInfo($info); - - // 下载图片 + 上传 + 入库(图片失败不影响主记录) - $imagesArray = json_decode($info['images'] ?? '[]', true); - if (json_last_error() !== JSON_ERROR_NONE) { - $imagesArray = []; - } - $saveDir = public_path('uploads/usedcar/' . $info['used_car_id']); - $image = $this->uploadAndRemove($imagesArray, $saveDir); - $info['image'] = $image['image']; - $info['image_pull'] = $image['image_pull']; - - // 继续派发配置页任务 -// if (!empty($info['car_id_url'])) { -// $this->pushUrl($info['car_id_url'], 'config', $insertId); -// } - - return $info; - } - /** * 抓取汽车之家数据 * @param Request $request From abb7ce89192e532dce16d81523a34dc975bb3004 Mon Sep 17 00:00:00 2001 From: "WIN-C27C5MSVTPO\\16961" <1696136552@qq.com> Date: Fri, 23 Jan 2026 10:53:15 +0800 Subject: [PATCH 07/10] =?UTF-8?q?=E4=BB=B7=E6=A0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/common/utils/crawl/service/UsedCarCrawlSingle.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/common/utils/crawl/service/UsedCarCrawlSingle.php b/app/common/utils/crawl/service/UsedCarCrawlSingle.php index 4e079ce..59b88da 100644 --- a/app/common/utils/crawl/service/UsedCarCrawlSingle.php +++ b/app/common/utils/crawl/service/UsedCarCrawlSingle.php @@ -253,7 +253,7 @@ class UsedCarCrawlSingle } // $cars['price'] = explode('万', $cars['price'])[0]??0; $cars['price'] = str_replace(['¥', '¥', '万'], '', $cars['price']); - $cars['price'] = $cars['price']*10000; + $cars['price'] = is_numeric($cars['price'])?$cars['price']*10000:$cars['price']; //$cars['price'] = $ql->find('#overlayPrice')->text(); $cars['title'] = $cars['product_name']; $cars['condition_desc'] = $ql->find('div.leave-message-box #messageBox')->text(); From 9b0fbecaa0286c3621b5fd6589beaf1d1f7f0123 Mon Sep 17 00:00:00 2001 From: "WIN-C27C5MSVTPO\\16961" <1696136552@qq.com> Date: Fri, 23 Jan 2026 11:49:07 +0800 Subject: [PATCH 08/10] =?UTF-8?q?html=E7=BC=96=E7=A0=81=E8=BD=AC=E6=8D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../crawl/service/UsedCarCrawlSingle.php | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/app/common/utils/crawl/service/UsedCarCrawlSingle.php b/app/common/utils/crawl/service/UsedCarCrawlSingle.php index 59b88da..2f3e55e 100644 --- a/app/common/utils/crawl/service/UsedCarCrawlSingle.php +++ b/app/common/utils/crawl/service/UsedCarCrawlSingle.php @@ -227,6 +227,9 @@ class UsedCarCrawlSingle throw new \Exception('无法获取页面 HTML'); } + // che168 页面为 gb2312,Linux 默认 UTF-8,直接解析会导致 DOM 解析异常。统一转 UTF-8 再解析。 + $html = $this->ensureUtf8ForChe168($html); + // 使用 QueryList 解析 HTML $ql = QueryList::html($html); @@ -659,6 +662,32 @@ class UsedCarCrawlSingle return null; } + /** + * 将 che168 页面 (gb2312/gbk) 转为 UTF-8,避免 Linux 下 DOM 解析异常 + * + * @param string $html + * @return string + */ + protected function ensureUtf8ForChe168(string $html): string + { + if (!function_exists('mb_convert_encoding')) { + return $html; + } + $isGb = stripos($html, 'charset=gb2312') !== false || stripos($html, 'charset=gbk') !== false; + if (!$isGb) { + return $html; + } + $out = @mb_convert_encoding($html, 'UTF-8', 'GB2312'); + if ($out === false || $out === '') { + $out = @mb_convert_encoding($html, 'UTF-8', 'GBK'); + } + if ($out !== false && $out !== '') { + $html = $out; + $html = preg_replace('/charset\s*=\s*["\']?gb(2312|k)["\']?/i', 'charset="utf-8"', $html); + } + return $html; + } + /** * 安全解 JSON * From 25891f2106240d2e6796fd67e3ed7631056a7fcd Mon Sep 17 00:00:00 2001 From: "WIN-C27C5MSVTPO\\16961" <1696136552@qq.com> Date: Fri, 23 Jan 2026 13:35:12 +0800 Subject: [PATCH 09/10] =?UTF-8?q?=E5=9B=BE=E7=89=87=E4=B8=8B=E8=BD=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../utils/crawl/service/UsedCarCrawlSingle.php | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/app/common/utils/crawl/service/UsedCarCrawlSingle.php b/app/common/utils/crawl/service/UsedCarCrawlSingle.php index 2f3e55e..1459476 100644 --- a/app/common/utils/crawl/service/UsedCarCrawlSingle.php +++ b/app/common/utils/crawl/service/UsedCarCrawlSingle.php @@ -307,7 +307,7 @@ class UsedCarCrawlSingle $saveDir = public_path('uploads/usedcar/' . $carId); $cars['used_car_id'] = $carId; - $image = $this->uploadAndRemove($images, $saveDir); + $image = $this->uploadAndRemove($images, $saveDir, 'https://www.che168.com/'); $cars['image'] = $image['image']; $cars['image_pull'] = $image['image_pull']; @@ -320,14 +320,14 @@ class UsedCarCrawlSingle * @param array $images 原始相册URL列表 * @param string $path 本地保存目录 */ - public function uploadAndRemove(array $images, string $path) + public function uploadAndRemove(array $images, string $path, string $referer = '') { if (empty($images)) { return false; } // 下载图片 - $filePatch = $this->downloadImageAlbum($images, $path); + $filePatch = $this->downloadImageAlbum($images, $path,$referer); if (empty($filePatch)) { return false; } @@ -390,7 +390,7 @@ class UsedCarCrawlSingle * @param string $saveDir 保存目录 * @return array [原始URL => 保存路径或null] */ - public function downloadImageAlbum(array $urls, string $saveDir): array + public function downloadImageAlbum(array $urls, string $saveDir, string $referer): array { $result = []; @@ -405,7 +405,7 @@ class UsedCarCrawlSingle } try { - $path = $this->downloadImage($url, $saveDir); + $path = $this->downloadImage($url, $saveDir, $referer); } catch (Throwable $e) { $path = null; } @@ -420,7 +420,7 @@ class UsedCarCrawlSingle /** * 下载单张图片 */ - public function downloadImage(string $url, string $saveDir, ?string $filename = null, array $headers = []): ?string + public function downloadImage(string $url, string $saveDir, string $referer, ?string $filename = null, array $headers = []): ?string { if (!is_dir($saveDir)) { if (!@mkdir($saveDir, 0777, true) && !is_dir($saveDir)) { @@ -442,10 +442,12 @@ class UsedCarCrawlSingle $savePath = rtrim($saveDir, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . $filename; + $referer = empty($referer) ? self::DEFAULT_HEADERS['Referer'] : $referer; $defaultHeaders = [ 'User-Agent: ' . self::DEFAULT_HEADERS['User-Agent'], 'Accept: image/avif,image/webp,image/apng,image/*,*/*;q=0.8', - 'Referer: ' . self::DEFAULT_HEADERS['Referer'], +// 'Referer: ' . self::DEFAULT_HEADERS['Referer'], + 'Referer: ' . $referer, ]; foreach ($headers as $h) { From 68a27d645c0367bb11e8e3e15b758d4405106c19 Mon Sep 17 00:00:00 2001 From: "WIN-C27C5MSVTPO\\16961" <1696136552@qq.com> Date: Fri, 23 Jan 2026 13:55:52 +0800 Subject: [PATCH 10/10] =?UTF-8?q?=E5=9B=BE=E7=89=87=E6=8A=93=E5=8F=96?= =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../crawl/service/UsedCarCrawlSingle.php | 51 +++++++++++-------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/app/common/utils/crawl/service/UsedCarCrawlSingle.php b/app/common/utils/crawl/service/UsedCarCrawlSingle.php index 1459476..604ff4b 100644 --- a/app/common/utils/crawl/service/UsedCarCrawlSingle.php +++ b/app/common/utils/crawl/service/UsedCarCrawlSingle.php @@ -278,28 +278,13 @@ class UsedCarCrawlSingle $year = end($info); $cars['year'] = !empty($year) ? str_replace('款', '', $year):''; - //图片 - $images = $ql->find('.swiper-container2 .swiper-slide .jiaodianphotoclick img')->map(function ($img) { - // 优先尝试 data-original(懒加载图片) - $src = $img->attr('src'); - - // 如果没有 data-original,尝试 src - if (!$src || $src === '' ||str_contains($src, 'default-che168')) { - $src = $img->attr('data-original'); - } - - // 补全 https - if (str_starts_with($src, '//')) { - $src = 'https:' . $src; - } - - // 过滤掉占位符图片 - if (str_contains($src, 'default-che168')) { - return null; - } - - return $src; - })->filter()->values()->all(); + // 图片:主用轮播 .swiper-container2,备用 #pic_li(懒加载 data-original) + $images = $this->collectChe168Images($ql, '.swiper-container2 .swiper-slide .jiaodianphotoclick img', false); + if (empty($images)) { + $images = $this->collectChe168Images($ql, '#pic_li img', true); + } + $images = array_values(array_unique($images)); + var_dump($images); if (empty($images)) { Log::warning("crawlChe168Car:未找到任何车辆图片,URL: {$url}"); @@ -664,6 +649,28 @@ class UsedCarCrawlSingle return null; } + /** + * 从 che168 页面上收集图片 URL + * + * @param \QL\QueryList $ql + * @param string $selector 选择器 + * @param bool $preferDataOriginal 为 true 时优先用 data-original(#pic_li 懒加载) + * @return array + */ + protected function collectChe168Images($ql, string $selector, bool $preferDataOriginal): array + { + return $ql->find($selector)->map(function ($img) use ($preferDataOriginal) { + $src = $preferDataOriginal ? ($img->attr('data-original') ?: $img->attr('src')) : ($img->attr('src') ?: $img->attr('data-original')); + if (!$src || $src === '' || str_contains($src, 'default-che168')) { + return null; + } + if (str_starts_with($src, '//')) { + $src = 'https:' . $src; + } + return $src; + })->filter()->values()->all(); + } + /** * 将 che168 页面 (gb2312/gbk) 转为 UTF-8,避免 Linux 下 DOM 解析异常 *