'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'Accept-Language' => 'zh-CN,zh;q=0.9', 'Referer' => 'https://www.dongchedi.com/', ]; /** * 获取网页HTML内容 * * @param string $url * @return string * @throws GuzzleException */ public function fetchHtml(string $url): string { $client = new Client([ 'verify' => false, ]); $response = $client->request('GET', $url, [ 'headers' => self::DEFAULT_HEADERS, 'timeout' => 10, ]); return (string)$response->getBody(); } /** * 获取懂车帝移动端分享短链最终url * * @param string $url * @return string * @throws GuzzleException */ public function getFinaUrl(string $url): string { $client = new Client([ 'verify' => false, ]); $response = $client->get($url, [ 'allow_redirects' => ['track_redirects' => true], ]); $history = $response->getHeader('X-Guzzle-Redirect-History'); $finalUrl = end($history); $pcUrl = $this->getPcUrl($finalUrl); return $pcUrl; } /** * 从移动端url获取sku_id * @param string $url * @return string|null */ public function getPcUrl(string $url): ?string { $query = parse_url($url, PHP_URL_QUERY); if (!$query) { return null; } parse_str($query, $params); return !empty($params['sku_id']) ? 'https://www.dongchedi.com/usedcar/'.$params['sku_id'] : null; } /** * 抓取懂车帝二手车信息 * @param string $url * @return array * @throws GuzzleException */ public function crawlCar(string $url){ $html = $this->fetchHtml($url); if ($html === '' || str_contains($html, '404 Not Found')) { throw new BusinessException("页面内容为空: {$url}"); } $info = $this->parseUsedCarDetail($html, $url); // $insertId = $this->saveUsedCarBasicInfo($info); // 下载图片 + 上传 + 入库(图片失败不影响主记录) $imagesArray = json_decode($info['images'] ?? '[]', true); if (json_last_error() !== JSON_ERROR_NONE) { $imagesArray = []; } $saveDir = public_path('uploads/usedcar/' . $info['used_car_id']); $image = $this->uploadAndRemove($imagesArray, $saveDir); $info['image'] = $image['image']; $info['image_pull'] = $image['image_pull']; // 继续派发配置页任务 // if (!empty($info['car_id_url'])) { // $this->pushUrl($info['car_id_url'], 'config', $insertId); // } return $info; } /** * 通过汽车之家分享链接获取pc端地址 * * @param string $url * @return string * @throws GuzzleException */ public function get168FinaUrl(string $url): string { $client = new Client([ 'verify' => false, 'timeout' => 10, 'headers' => [ 'User-Agent' => 'Mozilla/5.0 (Linux; Android 13; Mobile)' ] ]); $response = $client->get($url); $html = (string) $response->getBody(); preg_match( '/var\s+oUrl\s*=\s*[\'"]([^\'"]+che168\.com[^\'"]+)[\'"]/', $html, $m ); if(empty($m[1])){ //如果没有获取到移动端地址 throw new \Exception('168移动端地址解析失败'); } $pcUrl = $this->getChe168UrlByMobile($m[1]); return $pcUrl; } /** * 根据移动端url获取汽车之家pc端url * * @param string $url * @return string * @throws GuzzleException */ public function getChe168UrlByMobile(string $url): string { // 解析 URL获取车辆id $parsedUrl = parse_url($url); $path = $parsedUrl['path']; // "/dealer/162138/56723550.html" // 使用 pathinfo 获取文件名 $pathParts = explode('/', $path); $filename = end($pathParts); // "56723550.html" $carId = pathinfo($filename, PATHINFO_FILENAME); // "56723550" $pcUrl = 'https://www.che168.com/dealer/111/' . $carId . '.html?pvareaid=0'; return $pcUrl; } /** * 根据汽车之家pc端url转换为另一种页面的汽车之家pc端url * * @param string $url * @return string * @throws GuzzleException */ public function getChe168UrlByPc(string $url): string { // 解析URL获取查询参数部分 $queryString = parse_url($url, PHP_URL_QUERY); // 将查询参数解析为数组 parse_str($queryString, $params); // 获取infoid $carId = $params['infoid'] ?? null; $pcUrl = 'https://www.che168.com/dealer/111/' . $carId . '.html?pvareaid=0'; return $pcUrl; } /** * 抓取汽车之家数据 * @param Request $request */ public function crawlChe168Car(string $url) { // 解析 URL获取车辆id $parsedUrl = parse_url($url); $path = $parsedUrl['path']; // "/dealer/162138/56723550.html" // 使用 pathinfo 获取文件名 $pathParts = explode('/', $path); $filename = end($pathParts); // "56723550.html" $carId = pathinfo($filename, PATHINFO_FILENAME); // "56723550" // 初始化 Guzzle 客户端 $client = new Client([ 'headers' => [ 'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)', 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Referer' => 'https://www.che168.com/', ], 'verify' => false, 'timeout' => 10, ]); // 请求 HTML $response = $client->get($url); $html = (string)$response->getBody(); file_put_contents(runtime_path('craw.html'), $html); // 检查是否获取到 HTML if (empty($html)) { throw new \Exception('无法获取页面 HTML'); } // che168 页面为 gb2312,Linux 默认 UTF-8,直接解析会导致 DOM 解析异常。统一转 UTF-8 再解析。 $html = $this->ensureUtf8ForChe168($html); // 使用 QueryList 解析 HTML $ql = QueryList::html($html); // 解析车辆列表,每辆车可能在 li.cards-li 或类似类名里 $cars = $ql->find('div#nav1 ul.basic-item-ul li')->map(function($item) { $attribute = []; $field = $item->find('.item-name')->text(); $value = $item->text(); $title = ['product_name' => '产品名称','plate_time' => '上牌时间', 'mileage' => '表显里程', 'capacity' => '排       量', 'source_place'=> '所  在  地', 'body_color'=>'车身颜色','condition_desc'=>'车况介绍',]; $title_key = array_flip($title); if(in_array($field,$title)){ $attribute[$title_key[$field]] = str_replace($field, '', $value); } return $attribute; })->all(); $cars = array_filter($cars); $cars = array_merge(...$cars); $cars['product_name'] = $cars['car_name'] = $ql->find('div.car-box .car-brand-name')->text(); $cars['price'] = $ql->find('div.goodstartmoney')->text(); if(empty($cars['price'])){ $cars['price'] = $ql->find('span#overlayPrice')->text(); } // $cars['price'] = explode('万', $cars['price'])[0]??0; $cars['price'] = str_replace(['¥', '¥', '万'], '', $cars['price']); $cars['price'] = is_numeric($cars['price'])?$cars['price']*10000:$cars['price']; //$cars['price'] = $ql->find('#overlayPrice')->text(); $cars['title'] = $cars['product_name']; $cars['condition_desc'] = $ql->find('div.leave-message-box #messageBox')->text(); $cars['usage_mileage'] = str_replace('万公里', '', $cars['mileage']); //品牌与车系 $links = $ql->find('.bread-crumbs a')->map(function ($a) { return trim($a->text()); })->all(); $cars['brand_name'] = !empty($links[3]) ? str_replace('二手','',$links[3]): null; $cars['brand_id'] = 0; $cars['series_name'] = !empty($links[4]) ?str_replace('二手','',$links[4]): null; $cars['series_id'] = 0; $cars['car_id'] = 0; $info = explode(' ',$cars['product_name']); $info = array_filter($info, function($item) { return str_contains($item, '款'); }); $year = end($info); $cars['year'] = !empty($year) ? str_replace('款', '', $year):''; //图片 $images = $ql->find('.swiper-container2 .swiper-slide .jiaodianphotoclick img')->map(function ($img) { // 优先尝试 data-original(懒加载图片) $src = $img->attr('src'); // 如果没有 data-original,尝试 src if (!$src || $src === '' ||str_contains($src, 'default-che168')) { $src = $img->attr('data-original'); } // 补全 https if (str_starts_with($src, '//')) { $src = 'https:' . $src; } // 过滤掉占位符图片 if (str_contains($src, 'default-che168')) { return null; } return $src; })->filter()->values()->all(); if (empty($images)) { Log::warning("crawlChe168Car:未找到任何车辆图片,URL: {$url}"); } $saveDir = public_path('uploads/usedcar/' . $carId); $cars['used_car_id'] = $carId; $image = $this->uploadAndRemove($images, $saveDir); $cars['image'] = $image['image']; $cars['image_pull'] = $image['image_pull']; return $cars; } /** * 下载 -> 上传 -> 删除本地 * * @param array $images 原始相册URL列表 * @param string $path 本地保存目录 */ public function uploadAndRemove(array $images, string $path) { if (empty($images)) { return false; } // 下载图片 $filePatch = $this->downloadImageAlbum($images, $path); if (empty($filePatch)) { return false; } $uploadService = new UploadService(); $image = []; $image_pull = []; foreach ($filePatch as $url => $filePath) { if ($filePath === null) { continue; } try { // 上传云端 $param = [ 'path' => $filePath, 'scene' => 'usedcar_crawl', 'category' => 'usedcar', ]; $result = $uploadService->uploadLocalFile($param); if (empty($result['file_name'])) { // 上传失败则跳过,但不抛异常影响主流程 continue; } // 保存图片信息 // UsedcarCrawlResultsImages::create([ // 'uc_id' => $id, // 'used_car_id' => $usedCarId, // 'original_name'=> $result['original_name'] ?? '', // 'file_path' => $result['file_name'], // ]); $image[] = $result['file_name']; $image_pull[] = $result['file_url']; } catch (Throwable $e) { Log::error('crawlUsedCarImage'.$e->getMessage().'|'.$e->getLine().$e->getFile()); // 单张图失败不影响其他 // 可以按需写日志 } finally { // 删除本地文件 if (is_file($filePath)) { @unlink($filePath); } } } // 删除目录 rmdir($path); return ['image' => $image, 'image_pull' => $image_pull]; } /** * 批量下载相册 * * @param array $urls 图片URL数组 * @param string $saveDir 保存目录 * @return array [原始URL => 保存路径或null] */ public function downloadImageAlbum(array $urls, string $saveDir): array { $result = []; foreach ($urls as $url) { $url = trim((string)$url); if ($url === '') { continue; } if (strpos($url, '//') === 0) { $url = 'https:' . $url; } try { $path = $this->downloadImage($url, $saveDir); } catch (Throwable $e) { $path = null; } $result[$url] = $path; } return $result; } /** * 下载单张图片 */ public function downloadImage(string $url, string $saveDir, ?string $filename = null, array $headers = []): ?string { if (!is_dir($saveDir)) { if (!@mkdir($saveDir, 0777, true) && !is_dir($saveDir)) { // echo "[Image] 创建目录失败:{$saveDir}\n"; return null; } } if ($filename === null) { $path = parse_url($url, PHP_URL_PATH); $ext = pathinfo($path ?? '', PATHINFO_EXTENSION); if ($ext === '') { $ext = 'jpg'; } $filename = md5($url) . '.' . $ext; } $savePath = rtrim($saveDir, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . $filename; $defaultHeaders = [ 'User-Agent: ' . self::DEFAULT_HEADERS['User-Agent'], 'Accept: image/avif,image/webp,image/apng,image/*,*/*;q=0.8', 'Referer: ' . self::DEFAULT_HEADERS['Referer'], ]; foreach ($headers as $h) { $defaultHeaders[] = $h; } $ch = curl_init($url); curl_setopt_array($ch, [ CURLOPT_RETURNTRANSFER => true, CURLOPT_FOLLOWLOCATION => true, CURLOPT_CONNECTTIMEOUT => 5, CURLOPT_TIMEOUT => 15, CURLOPT_SSL_VERIFYPEER => false, CURLOPT_SSL_VERIFYHOST => false, CURLOPT_HTTPHEADER => $defaultHeaders, ]); $data = curl_exec($ch); $errNo = curl_errno($ch); $code = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE); curl_close($ch); if ($errNo !== 0 || $code !== 200 || $data === false || $data === '') { return null; } $bytes = @file_put_contents($savePath, $data); if ($bytes === false) { return null; } return $savePath; } /** * 解析二手车详情页信息 * * @param string $html * @param string $url * @return array * @throws BusinessException */ public function parseUsedCarDetail(string $html = '', string $url = ''): array { try { $ql = QueryList::html($html); } catch (Throwable $e) { throw new BusinessException('解析 HTML 失败(详情页): ' . $e->getMessage()); } $title = trim($ql->find('h1')->text()); $price = trim($ql->find('.head-info_price-wrap__Y4bxi > p')->text()); $pageText = $ql->find('body')->text(); $jsonText = $ql->find('#__NEXT_DATA__')->text(); if ($jsonText === '') { throw new BusinessException('详情页未找到 __NEXT_DATA__ JSON'); } $jsonArray = $this->decodeJson($jsonText, '详情页 __NEXT_DATA__'); // 车辆相册 $images = $jsonArray['props']['pageProps']['skuDetail']['head_images'] ?? []; // 车辆ID / 年份 $carInfo = $jsonArray['props']['pageProps']['skuDetail']['car_info'] ?? []; $carId = $carInfo['car_id'] ?? 0; $year = (int)($carInfo['year'] ?? 0); // 车辆ID URL $carIdUrl = $carId ? ('https://www.dongchedi.com/auto/params-carIds-' . $carId) : ''; // 能源类型(两个字段择一) $overview = $jsonArray['props']['pageProps']['skuDetail']['car_config_overview'] ?? []; $fuelForm = $overview['power']['fuel_form'] ?? ''; $horsepower = $overview['power']['horsepower'] ?? ''; //马力 $capacity = $overview['power']['capacity'] ?? ''; //排量 $newFuelForm = $overview['new_energy_power']['fuel_form'] ?? ''; $resultFuelForm = $fuelForm === '' ? $newFuelForm : $fuelForm; // 使用里程 $importantText = $jsonArray['props']['pageProps']['skuDetail']['important_text'] ?? ''; $importantTextArray = $importantText ? explode(' | ', $importantText) : []; $usageMileage = ''; if (isset($importantTextArray[1])) { $usageMileage = str_replace('万公里', '', $importantTextArray[1]); } // 车辆名称 品牌、车系、车型 $brandName = $carInfo['brand_name'] ?? ''; $brand_id = $carInfo['brand_id'] ?? ''; $seriesName = $carInfo['series_name'] ?? ''; $series_id = $carInfo['series_id'] ?? ''; $carName = $carInfo['car_name'] ?? ''; $bodyColor = $carInfo['body_color'] ?? ''; //颜色 // 商家信息 $shopInfo = $jsonArray['props']['pageProps']['skuDetail']['shop_info'] ?? []; $formatShopInfo = [ 'shop_id' => $shopInfo['shop_id'] ?? 0, 'shop_name' => $shopInfo['shop_short_name'] ?? '', 'shop_address' => $shopInfo['shop_address'] ?? '', ]; // 简介参数 $otherParams = $jsonArray['props']['pageProps']['skuDetail']['other_params'] ?? []; $displacement = $this->getParamValue($otherParams, '排量'); $gearbox = $this->getParamValue($otherParams, '变速箱'); $sourcePlace = $this->getParamValue($otherParams, '车源地'); $plateTime = $this->getParamValue($otherParams, '上牌时间'); $formatCarInfo = [ 'displacement' => $displacement, 'gearbox' => $gearbox, ]; // 提取新车指导价 / 比新车省金额 $newPrice = $this->extractPatternValue($pageText, '/新车指导价[::]\s*([^\r\n万]*万)/u'); $priceDifference = $this->extractPatternValue($pageText, '/比新车省[::]\s*([^\r\n万]*万)/u'); // 提取车况介绍 $conditionDesc = ''; if ($pageText && preg_match('/车况介绍\s*(.*?)\s*推荐理由/su', $pageText, $m)) { $conditionDesc = trim($m[1]); } return [ 'used_car_id' => $this->extractIdFromUrl($url), 'car_id' => $carId, 'url' => $url, 'title' => $title, 'price' => str_replace('万', '', $this->decodePrice($price)), 'new_price' => str_replace('万', '', $newPrice), 'price_difference' => str_replace('万', '', $priceDifference), 'condition_desc' => $conditionDesc, 'images' => json_encode($images, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), 'car_id_url' => $carIdUrl, 'used_detail' => $jsonText, 'format_shop_info' => json_encode($formatShopInfo, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), 'fuel_form' => $resultFuelForm, 'usage_mileage' => $usageMileage, 'brand_name' => $brandName, 'series_name' => $seriesName, 'car_name' => $carName, 'year' => $year, 'format_car_info' => json_encode($formatCarInfo, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), 'body_color' => $bodyColor, 'horsepower' => $horsepower, 'capacity' => $capacity, 'source_place' => $sourcePlace, 'brand_id' => $brand_id, 'series_id' => $series_id, 'plate_time' => $plateTime, ]; } /** * 解码懂车帝价格 */ protected function decodePrice(string $str): string { $map = [ "\u{E53D}" => '1', "\u{E3F0}" => '2', "\u{E422}" => '3', "\u{E42C}" => '4', "\u{E49C}" => '5', "\u{E42B}" => '6', "\u{E4FE}" => '7', "\u{E548}" => '8', "\u{E4C8}" => '9', "\u{E453}" => '0', "\u{E45F}" => '万', ]; return strtr($str, $map); } /** * 从URL中提取ID */ protected function extractIdFromUrl(string $url) { if (preg_match('/\/(\d+)$/', $url, $matches)) { return (int)$matches[1]; } return false; } /** * 提取符合正则表达式的值(只保留数字和小数点) */ protected function extractPatternValue(string $text, string $pattern): string { $value = ''; if ($text && preg_match($pattern, $text, $m)) { $value = trim(preg_replace('/[^0-9.]/', '', $m[1])); } return $value; } /** * 从参数列表中获取指定名称的值 */ public function getParamValue(array $params, string $name): ?string { foreach ($params as $item) { if (($item['name'] ?? null) === $name) { return $item['value'] ?? null; } } return null; } /** * 将 che168 页面 (gb2312/gbk) 转为 UTF-8,避免 Linux 下 DOM 解析异常 * * @param string $html * @return string */ protected function ensureUtf8ForChe168(string $html): string { if (!function_exists('mb_convert_encoding')) { return $html; } $isGb = stripos($html, 'charset=gb2312') !== false || stripos($html, 'charset=gbk') !== false; if (!$isGb) { return $html; } $out = @mb_convert_encoding($html, 'UTF-8', 'GB2312'); if ($out === false || $out === '') { $out = @mb_convert_encoding($html, 'UTF-8', 'GBK'); } if ($out !== false && $out !== '') { $html = $out; $html = preg_replace('/charset\s*=\s*["\']?gb(2312|k)["\']?/i', 'charset="utf-8"', $html); } return $html; } /** * 安全解 JSON * * @param string $json * @param string $context * @return array * @throws BusinessException */ protected function decodeJson(string $json, string $context = 'JSON'): array { $data = json_decode($json, true); if (json_last_error() !== JSON_ERROR_NONE || !is_array($data)) { throw new BusinessException($context . ' 解析失败:' . json_last_error_msg()); } return $data; } }