diff --git a/app/common/controller/UsedCarController.php b/app/common/controller/UsedCarController.php index b0d8bd8..b9a117c 100644 --- a/app/common/controller/UsedCarController.php +++ b/app/common/controller/UsedCarController.php @@ -11,6 +11,7 @@ use app\common\service\UsedCarService; use app\common\utils\crawl\service\UsedCarCrawlSingle; use app\common\utils\pdf\service\MpdfDriver; use support\exception\BusinessException; +use support\Log; use support\Request; class UsedCarController extends BaseController @@ -128,6 +129,25 @@ class UsedCarController extends BaseController return $data; } + /** + * 移动端链接抓取 + * @param Request $request + */ + public function crawlMobileCar(Request $request) + { + try { + $urlShare = $request->get('url'); + + $finalUrl = (new UsedCarCrawlSingle())->getFinaUrl($urlShare); + + $request->setGet('url', $finalUrl); + $data = $this->crawlCar($request); + return $data; + } catch (\Throwable $e) { + return $this->error($e->getMessage()); + } + } + public function crawlCar(Request $request) { try { @@ -135,15 +155,16 @@ class UsedCarController extends BaseController if (empty($url)){ throw new BusinessException('url不能为空'); } - $info = (new UsedCarCrawlSingle())->crawlCar($url); + //$info = (new UsedCarCrawlSingle())->crawlCar($url); + $info = $this->service->selectCraw($url); $data = [ 'user_id' => $request->user['id'], 'product_name' => $info['title'], 'price' => $info['price']<200 ? $info['price']*10000 : $info['price'], 'source_place' => $info['source_place'], - 'displacement' => $info['capacity'], //排量 + 'displacement' => $info['capacity']??'', //排量 'mileage' => $info['usage_mileage']*10000, //里程 - 'power' => $info['horsepower'], //马力 + 'power' => $info['horsepower']??'', //马力 'body_color' => $info['body_color'], 'images' => $info['image'], 'image_pull' => $info['image_pull'], @@ -166,7 +187,8 @@ class UsedCarController extends BaseController if (str_contains($e->getMessage(), '404 Not Found')){ return $this->error('车辆已下架或找不到页面'); } - return $this->error($e->getMessage()); + Log::error('crawlCar fail',['param' => $request->get('url'),'message' => $e->getMessage().$e->getFile().$e->getLine()]); + return $this->error($e->getMessage().$e->getFile().$e->getLine()); } } diff --git a/app/common/service/UsedCarService.php b/app/common/service/UsedCarService.php index 6433770..6a20828 100644 --- a/app/common/service/UsedCarService.php +++ b/app/common/service/UsedCarService.php @@ -2,10 +2,12 @@ namespace app\common\service; +use app\common\utils\crawl\service\UsedCarCrawlSingle; use app\common\validate\UsedCarValidate; use app\common\traits\crud\CrudServiceTrait; use app\common\model\UsedCar; use GuzzleHttp\Client; +use GuzzleHttp\Exception\GuzzleException; use support\exception\BusinessException; use support\Log; use support\Request; @@ -422,4 +424,50 @@ class UsedCarService return $path; } + + /** + * 选择车辆抓取方式 + * @param string $url + */ + public function selectCraw(string $url){ + $usedCarCrawl = new UsedCarCrawlSingle(); + switch (true){ + case str_contains($url,'dongchedi.com'): //懂车帝pc端 + $info = $usedCarCrawl->crawlCar($url); + break; + case str_contains($url,'dcdapp.com'): //懂车帝移动端 + $finalUrl = $usedCarCrawl->getPcUrl($url); //获取pc端页面链接 + if(empty($finalUrl)){ + throw new \Exception('链接解析错误'); + } + $info = $usedCarCrawl->crawlCar($finalUrl); + break; + case str_contains($url,'t.zijieimg.com'): //懂车帝移动端分享链接 + $finalUrl = $usedCarCrawl->getFinaUrl($url); //解析抖音短链,并获取pc端页面链接 + if(empty($finalUrl)){ + throw new \Exception('链接解析错误'); + } + $info = $usedCarCrawl->crawlCar($finalUrl); + break; + case str_contains($url,'www.che168.com/dealer'): //汽车之家 + $info = $usedCarCrawl->crawlChe168Car($url); + break; + case str_contains($url,'www.che168.com/home?infoid='): //汽车之家pc端链接转换 + $url = $usedCarCrawl->getChe168UrlByPc($url); + $info = $usedCarCrawl->crawlChe168Car($url); + break; + case str_contains($url,'wap.che168.com/dealer'): //汽车之家移动端链接 + $finalUrl = $usedCarCrawl->getChe168UrlByMobile($url); + $info = $usedCarCrawl->crawlChe168Car($finalUrl); + break; + case str_contains($url,'comm.app.autohome.com.cn'): //汽车之家移动端链接 + $finalUrl = $usedCarCrawl->get168FinaUrl($url); + $info = $usedCarCrawl->crawlChe168Car($finalUrl); + break; + default: + Log::info('未知链接:'.$url); + throw new \Exception('加紧开发中'); + } + return $info; + } } diff --git a/app/common/utils/crawl/service/UsedCarCrawlSingle.php b/app/common/utils/crawl/service/UsedCarCrawlSingle.php index c04cd10..604ff4b 100644 --- a/app/common/utils/crawl/service/UsedCarCrawlSingle.php +++ b/app/common/utils/crawl/service/UsedCarCrawlSingle.php @@ -36,10 +36,55 @@ class UsedCarCrawlSingle 'headers' => self::DEFAULT_HEADERS, 'timeout' => 10, ]); - return (string)$response->getBody(); } + /** + * 获取懂车帝移动端分享短链最终url + * + * @param string $url + * @return string + * @throws GuzzleException + */ + public function getFinaUrl(string $url): string + { + $client = new Client([ + 'verify' => false, + ]); + $response = $client->get($url, [ + 'allow_redirects' => ['track_redirects' => true], + ]); + + $history = $response->getHeader('X-Guzzle-Redirect-History'); + $finalUrl = end($history); + + $pcUrl = $this->getPcUrl($finalUrl); + return $pcUrl; + } + + /** + * 从移动端url获取sku_id + * @param string $url + * @return string|null + */ + public function getPcUrl(string $url): ?string + { + $query = parse_url($url, PHP_URL_QUERY); + if (!$query) { + return null; + } + + parse_str($query, $params); + + return !empty($params['sku_id']) ? 'https://www.dongchedi.com/usedcar/'.$params['sku_id'] : null; + } + + /** + * 抓取懂车帝二手车信息 + * @param string $url + * @return array + * @throws GuzzleException + */ public function crawlCar(string $url){ $html = $this->fetchHtml($url); if ($html === '' || str_contains($html, '404 Not Found')) { @@ -68,20 +113,206 @@ class UsedCarCrawlSingle return $info; } + /** + * 通过汽车之家分享链接获取pc端地址 + * + * @param string $url + * @return string + * @throws GuzzleException + */ + public function get168FinaUrl(string $url): string + { + $client = new Client([ + 'verify' => false, + 'timeout' => 10, + 'headers' => [ + 'User-Agent' => 'Mozilla/5.0 (Linux; Android 13; Mobile)' + ] + ]); + + $response = $client->get($url); + $html = (string) $response->getBody(); + + preg_match( + '/var\s+oUrl\s*=\s*[\'"]([^\'"]+che168\.com[^\'"]+)[\'"]/', + $html, + $m + ); + + if(empty($m[1])){ //如果没有获取到移动端地址 + throw new \Exception('168移动端地址解析失败'); + } + + $pcUrl = $this->getChe168UrlByMobile($m[1]); + return $pcUrl; + } + + /** + * 根据移动端url获取汽车之家pc端url + * + * @param string $url + * @return string + * @throws GuzzleException + */ + public function getChe168UrlByMobile(string $url): string + { + + // 解析 URL获取车辆id + $parsedUrl = parse_url($url); + $path = $parsedUrl['path']; // "/dealer/162138/56723550.html" + // 使用 pathinfo 获取文件名 + $pathParts = explode('/', $path); + $filename = end($pathParts); // "56723550.html" + $carId = pathinfo($filename, PATHINFO_FILENAME); // "56723550" + + $pcUrl = 'https://www.che168.com/dealer/111/' . $carId . '.html?pvareaid=0'; + return $pcUrl; + } + + /** + * 根据汽车之家pc端url转换为另一种页面的汽车之家pc端url + * + * @param string $url + * @return string + * @throws GuzzleException + */ + public function getChe168UrlByPc(string $url): string + { + // 解析URL获取查询参数部分 + $queryString = parse_url($url, PHP_URL_QUERY); + // 将查询参数解析为数组 + parse_str($queryString, $params); + + // 获取infoid + $carId = $params['infoid'] ?? null; + + $pcUrl = 'https://www.che168.com/dealer/111/' . $carId . '.html?pvareaid=0'; + + return $pcUrl; + } + + /** + * 抓取汽车之家数据 + * @param Request $request + */ + public function crawlChe168Car(string $url) + { + // 解析 URL获取车辆id + $parsedUrl = parse_url($url); + $path = $parsedUrl['path']; // "/dealer/162138/56723550.html" + // 使用 pathinfo 获取文件名 + $pathParts = explode('/', $path); + $filename = end($pathParts); // "56723550.html" + $carId = pathinfo($filename, PATHINFO_FILENAME); // "56723550" + + // 初始化 Guzzle 客户端 + $client = new Client([ + 'headers' => [ + 'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)', + 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Referer' => 'https://www.che168.com/', + ], + 'verify' => false, + 'timeout' => 10, + ]); + + + // 请求 HTML + $response = $client->get($url); + $html = (string)$response->getBody(); + file_put_contents(runtime_path('craw.html'), $html); + + // 检查是否获取到 HTML + if (empty($html)) { + throw new \Exception('无法获取页面 HTML'); + } + + // che168 页面为 gb2312,Linux 默认 UTF-8,直接解析会导致 DOM 解析异常。统一转 UTF-8 再解析。 + $html = $this->ensureUtf8ForChe168($html); + + // 使用 QueryList 解析 HTML + $ql = QueryList::html($html); + + // 解析车辆列表,每辆车可能在 li.cards-li 或类似类名里 + $cars = $ql->find('div#nav1 ul.basic-item-ul li')->map(function($item) { + $attribute = []; + $field = $item->find('.item-name')->text(); + $value = $item->text(); + $title = ['product_name' => '产品名称','plate_time' => '上牌时间', 'mileage' => '表显里程', 'capacity' => '排       量', 'source_place'=> '所  在  地', 'body_color'=>'车身颜色','condition_desc'=>'车况介绍',]; + $title_key = array_flip($title); + if(in_array($field,$title)){ + $attribute[$title_key[$field]] = str_replace($field, '', $value); + } + + return $attribute; + })->all(); + + $cars = array_filter($cars); + $cars = array_merge(...$cars); + $cars['product_name'] = $cars['car_name'] = $ql->find('div.car-box .car-brand-name')->text(); + $cars['price'] = $ql->find('div.goodstartmoney')->text(); + if(empty($cars['price'])){ + $cars['price'] = $ql->find('span#overlayPrice')->text(); + } +// $cars['price'] = explode('万', $cars['price'])[0]??0; + $cars['price'] = str_replace(['¥', '¥', '万'], '', $cars['price']); + $cars['price'] = is_numeric($cars['price'])?$cars['price']*10000:$cars['price']; + //$cars['price'] = $ql->find('#overlayPrice')->text(); + $cars['title'] = $cars['product_name']; + $cars['condition_desc'] = $ql->find('div.leave-message-box #messageBox')->text(); + $cars['usage_mileage'] = str_replace('万公里', '', $cars['mileage']); + + //品牌与车系 + $links = $ql->find('.bread-crumbs a')->map(function ($a) { + return trim($a->text()); + })->all(); + $cars['brand_name'] = !empty($links[3]) ? str_replace('二手','',$links[3]): null; + $cars['brand_id'] = 0; + $cars['series_name'] = !empty($links[4]) ?str_replace('二手','',$links[4]): null; + $cars['series_id'] = 0; + $cars['car_id'] = 0; + $info = explode(' ',$cars['product_name']); + $info = array_filter($info, function($item) { + return str_contains($item, '款'); + }); + $year = end($info); + $cars['year'] = !empty($year) ? str_replace('款', '', $year):''; + + // 图片:主用轮播 .swiper-container2,备用 #pic_li(懒加载 data-original) + $images = $this->collectChe168Images($ql, '.swiper-container2 .swiper-slide .jiaodianphotoclick img', false); + if (empty($images)) { + $images = $this->collectChe168Images($ql, '#pic_li img', true); + } + $images = array_values(array_unique($images)); + var_dump($images); + + if (empty($images)) { + Log::warning("crawlChe168Car:未找到任何车辆图片,URL: {$url}"); + } + + $saveDir = public_path('uploads/usedcar/' . $carId); + $cars['used_car_id'] = $carId; + $image = $this->uploadAndRemove($images, $saveDir, 'https://www.che168.com/'); + $cars['image'] = $image['image']; + $cars['image_pull'] = $image['image_pull']; + + return $cars; + } + /** * 下载 -> 上传 -> 删除本地 * * @param array $images 原始相册URL列表 * @param string $path 本地保存目录 */ - public function uploadAndRemove(array $images, string $path) + public function uploadAndRemove(array $images, string $path, string $referer = '') { if (empty($images)) { return false; } // 下载图片 - $filePatch = $this->downloadImageAlbum($images, $path); + $filePatch = $this->downloadImageAlbum($images, $path,$referer); if (empty($filePatch)) { return false; } @@ -144,7 +375,7 @@ class UsedCarCrawlSingle * @param string $saveDir 保存目录 * @return array [原始URL => 保存路径或null] */ - public function downloadImageAlbum(array $urls, string $saveDir): array + public function downloadImageAlbum(array $urls, string $saveDir, string $referer): array { $result = []; @@ -159,7 +390,7 @@ class UsedCarCrawlSingle } try { - $path = $this->downloadImage($url, $saveDir); + $path = $this->downloadImage($url, $saveDir, $referer); } catch (Throwable $e) { $path = null; } @@ -174,7 +405,7 @@ class UsedCarCrawlSingle /** * 下载单张图片 */ - public function downloadImage(string $url, string $saveDir, ?string $filename = null, array $headers = []): ?string + public function downloadImage(string $url, string $saveDir, string $referer, ?string $filename = null, array $headers = []): ?string { if (!is_dir($saveDir)) { if (!@mkdir($saveDir, 0777, true) && !is_dir($saveDir)) { @@ -196,10 +427,12 @@ class UsedCarCrawlSingle $savePath = rtrim($saveDir, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . $filename; + $referer = empty($referer) ? self::DEFAULT_HEADERS['Referer'] : $referer; $defaultHeaders = [ 'User-Agent: ' . self::DEFAULT_HEADERS['User-Agent'], 'Accept: image/avif,image/webp,image/apng,image/*,*/*;q=0.8', - 'Referer: ' . self::DEFAULT_HEADERS['Referer'], +// 'Referer: ' . self::DEFAULT_HEADERS['Referer'], + 'Referer: ' . $referer, ]; foreach ($headers as $h) { @@ -416,6 +649,54 @@ class UsedCarCrawlSingle return null; } + /** + * 从 che168 页面上收集图片 URL + * + * @param \QL\QueryList $ql + * @param string $selector 选择器 + * @param bool $preferDataOriginal 为 true 时优先用 data-original(#pic_li 懒加载) + * @return array + */ + protected function collectChe168Images($ql, string $selector, bool $preferDataOriginal): array + { + return $ql->find($selector)->map(function ($img) use ($preferDataOriginal) { + $src = $preferDataOriginal ? ($img->attr('data-original') ?: $img->attr('src')) : ($img->attr('src') ?: $img->attr('data-original')); + if (!$src || $src === '' || str_contains($src, 'default-che168')) { + return null; + } + if (str_starts_with($src, '//')) { + $src = 'https:' . $src; + } + return $src; + })->filter()->values()->all(); + } + + /** + * 将 che168 页面 (gb2312/gbk) 转为 UTF-8,避免 Linux 下 DOM 解析异常 + * + * @param string $html + * @return string + */ + protected function ensureUtf8ForChe168(string $html): string + { + if (!function_exists('mb_convert_encoding')) { + return $html; + } + $isGb = stripos($html, 'charset=gb2312') !== false || stripos($html, 'charset=gbk') !== false; + if (!$isGb) { + return $html; + } + $out = @mb_convert_encoding($html, 'UTF-8', 'GB2312'); + if ($out === false || $out === '') { + $out = @mb_convert_encoding($html, 'UTF-8', 'GBK'); + } + if ($out !== false && $out !== '') { + $html = $out; + $html = preg_replace('/charset\s*=\s*["\']?gb(2312|k)["\']?/i', 'charset="utf-8"', $html); + } + return $html; + } + /** * 安全解 JSON * diff --git a/config/route.php b/config/route.php index f6b0373..b9337f4 100644 --- a/config/route.php +++ b/config/route.php @@ -68,12 +68,11 @@ Route::group('/common', function () { Route::any('/destroy', [app\common\controller\UsedCarController::class, 'destroy']); Route::any('/crawl', [app\common\controller\UsedCarController::class, 'crawlKeyword']); Route::any('/url_crawl', [app\common\controller\UsedCarController::class, 'crawlCar']); + Route::any('/mobile/url_crawl', [app\common\controller\UsedCarController::class, 'crawlMobileCar']); }); Route::group('/user', function () { Route::any('/info', [app\common\controller\UserController::class, 'show']); Route::any('/edit', [app\common\controller\UserController::class, 'edit']); }); - - }); \ No newline at end of file