'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'Accept-Language' => 'zh-CN,zh;q=0.9', 'Referer' => 'https://www.dongchedi.com/', ]; // Redis队列键名 private const REDIS_QUEUE_KEY = 'usedcar:urls'; // 能源类型常量 public const ENERGY_TYPE_RENEWABLE = 'RE'; public const ENERGY_TYPE_CONVENTIONAL = 'CE'; public const ENERGY_TYPE_UNKNOWN = 'UNKNOWN'; /** * 插入队列url * * @param string $url URL地址 * @param string $scene 场景:usedcar(二手信息) config(配置信息) * @param int $id 记录ID * @return int */ public function pushUrl(string $url = '', string $scene = 'usedcar', int $id = 0): int { $data = [ 'scene' => $scene, 'url' => $url, 'id' => $id, ]; return Redis::lpush( self::REDIS_QUEUE_KEY, json_encode($data, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) ); } /** * 获取队列url * * @return array|null * @throws BusinessException */ public function popUrl(): ?array { $data = Redis::rPop(self::REDIS_QUEUE_KEY); if (!$data) { return null; } $task = json_decode($data, true); if (json_last_error() !== JSON_ERROR_NONE || !is_array($task)) { throw new BusinessException('任务数据解析失败:' . json_last_error_msg()); } // 基本字段校验 if (empty($task['url']) || empty($task['scene'])) { throw new BusinessException('队列任务数据缺少必要字段:url 或 scene'); } return $task; } /** * 执行抓取任务 * * @return bool * @throws BusinessException */ public function execute(): bool { try { $task = $this->popUrl(); if (!$task) { // 队列为空,任务结束 return false; } $url = (string)$task['url']; $scene = (string)$task['scene']; $id = (int)($task['id'] ?? 0); $html = $this->fetchHtml($url); if ($html === '') { throw new BusinessException("页面内容为空: {$url}"); } switch ($scene) { case 'usedcar': $info = $this->parseUsedCarDetail($html, $url); $insertId = $this->saveUsedCarBasicInfo($info); // 下载图片 + 上传 + 入库(图片失败不影响主记录) $imagesArray = json_decode($info['images'] ?? '[]', true); if (json_last_error() !== JSON_ERROR_NONE) { $imagesArray = []; } $saveDir = public_path('uploads/usedcar/' . $insertId); $this->uploadAndRemove($imagesArray, $saveDir, $info['used_car_id'], $insertId); // 继续派发配置页任务 if (!empty($info['car_id_url'])) { $this->pushUrl($info['car_id_url'], 'config', $insertId); } break; case 'config': if ($id <= 0) { throw new BusinessException("配置任务缺少主记录ID"); } $data = $this->parseConfigDetail($html); $this->saveUsedCarConfigInfo($data, $id); break; default: throw new BusinessException("未知的任务场景: " . $scene); } return true; } catch (GuzzleException $e) { throw new BusinessException("网络请求失败: " . $e->getMessage()); } catch (BusinessException $e) { // 业务异常原样抛出,方便上层知道是逻辑问题 throw $e; } catch (Throwable $th) { // 兜底异常,避免 worker 直接挂掉 throw new BusinessException( "执行任务出错: " . $th->getMessage() . ' | 行号: ' . $th->getLine() . ' | 文件: ' . $th->getFile() ); } } /** * 获取网页HTML内容 * * @param string $url * @return string * @throws GuzzleException */ public function fetchHtml(string $url): string { $client = new Client(); $response = $client->request('GET', $url, [ 'headers' => self::DEFAULT_HEADERS, 'timeout' => 10, ]); return (string)$response->getBody(); } /** * 解析二手车详情页信息 * * @param string $html * @param string $url * @return array * @throws BusinessException */ public function parseUsedCarDetail(string $html = '', string $url = ''): array { try { $ql = QueryList::html($html); } catch (Throwable $e) { throw new BusinessException('解析 HTML 失败(详情页): ' . $e->getMessage()); } $title = trim($ql->find('h1')->text()); $price = trim($ql->find('.head-info_price-wrap__Y4bxi > p')->text()); $pageText = $ql->find('body')->text(); $jsonText = $ql->find('#__NEXT_DATA__')->text(); if ($jsonText === '') { throw new BusinessException('详情页未找到 __NEXT_DATA__ JSON'); } $jsonArray = $this->decodeJson($jsonText, '详情页 __NEXT_DATA__'); // 车辆相册 $images = $jsonArray['props']['pageProps']['skuDetail']['head_images'] ?? []; // 车辆ID / 年份 $carInfo = $jsonArray['props']['pageProps']['skuDetail']['car_info'] ?? []; $carId = $carInfo['car_id'] ?? 0; $year = (int)($carInfo['year'] ?? 0); // 车辆ID URL $carIdUrl = $carId ? ('https://www.dongchedi.com/auto/params-carIds-' . $carId) : ''; // 能源类型(两个字段择一) $overview = $jsonArray['props']['pageProps']['skuDetail']['car_config_overview'] ?? []; $fuelForm = $overview['power']['fuel_form'] ?? ''; $newFuelForm = $overview['new_energy_power']['fuel_form'] ?? ''; $resultFuelForm = $fuelForm === '' ? $newFuelForm : $fuelForm; // 使用里程 $importantText = $jsonArray['props']['pageProps']['skuDetail']['important_text'] ?? ''; $importantTextArray = $importantText ? explode(' | ', $importantText) : []; $usageMileage = ''; if (isset($importantTextArray[1])) { $usageMileage = str_replace('万公里', '', $importantTextArray[1]); } // 车辆名称 品牌、车系、车型 $brandName = $carInfo['brand_name'] ?? ''; $seriesName = $carInfo['series_name'] ?? ''; $carName = $carInfo['car_name'] ?? ''; // 商家信息 $shopInfo = $jsonArray['props']['pageProps']['skuDetail']['shop_info'] ?? []; $formatShopInfo = [ 'shop_id' => $shopInfo['shop_id'] ?? 0, 'shop_name' => $shopInfo['shop_short_name'] ?? '', 'shop_address' => $shopInfo['shop_address'] ?? '', ]; // 简介参数 $otherParams = $jsonArray['props']['pageProps']['skuDetail']['other_params'] ?? []; $displacement = $this->getParamValue($otherParams, '排量'); $gearbox = $this->getParamValue($otherParams, '变速箱'); $formatCarInfo = [ 'displacement' => $displacement, 'gearbox' => $gearbox, ]; // 提取新车指导价 / 比新车省金额 $newPrice = $this->extractPatternValue($pageText, '/新车指导价[::]\s*([^\r\n万]*万)/u'); $priceDifference = $this->extractPatternValue($pageText, '/比新车省[::]\s*([^\r\n万]*万)/u'); // 提取车况介绍 $conditionDesc = ''; if ($pageText && preg_match('/车况介绍\s*(.*?)\s*推荐理由/su', $pageText, $m)) { $conditionDesc = trim($m[1]); } return [ 'used_car_id' => $this->extractIdFromUrl($url), 'car_id' => $carId, 'url' => $url, 'title' => $title, 'price' => str_replace('万', '', $this->decodePrice($price)), 'new_price' => str_replace('万', '', $newPrice), 'price_difference' => str_replace('万', '', $priceDifference), 'condition_desc' => $conditionDesc, 'images' => json_encode($images, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), 'car_id_url' => $carIdUrl, 'used_detail' => $jsonText, 'format_shop_info' => json_encode($formatShopInfo, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), 'fuel_form' => $resultFuelForm, 'classify_energy' => $this->detectEnergyType($resultFuelForm), 'usage_mileage' => $usageMileage, 'brand_name' => $brandName, 'series_name' => $seriesName, 'car_name' => $carName, 'year' => $year, 'format_car_info' => json_encode($formatCarInfo, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), ]; } /** * 解析配置详情页信息 * * @param string $html * @return array * @throws BusinessException */ public function parseConfigDetail(string $html = ''): array { try { $ql = QueryList::html($html); } catch (Throwable $e) { throw new BusinessException('解析 HTML 失败(配置页): ' . $e->getMessage()); } $jsonText = $ql->find('#__NEXT_DATA__')->text(); if ($jsonText === '') { throw new BusinessException('配置页未找到 __NEXT_DATA__ JSON'); } $raw = $this->decodeJson($jsonText, '配置页 __NEXT_DATA__'); $properties = $raw['props']['pageProps']['rawData']['properties'] ?? []; $carInfo = $raw['props']['pageProps']['rawData']['car_info'][0] ?? []; $valuesMap = $this->buildValuesMap($carInfo); $groups = $this->buildCarSpecGroups($properties, $valuesMap); return [ 'format_car_info' => [ 'market_time' => $this->getFieldValue($groups, 'market_time'), // 上市时间 'driver_form' => $this->getFieldValue($groups, 'driver_form'), // 驱动形式 'seat_count' => $this->getFieldValue($groups, 'seat_count'), // 座位数 'engine_description' => $this->getFieldValue($groups, 'engine_description'), // 发动机 'engine_max_power_horsepower' => $this->getFieldValue($groups, 'engine_max_power_horsepower'), // 发动机最大功率 'electric_description' => $this->getFieldValue($groups, 'electric_description'), // 电动机 'battery_capacity' => $this->getFieldValue($groups, 'battery_capacity'), // 电池容量 'battery_type' => $this->getFieldValue($groups, 'battery_type'), // 电池类型 'recharge_mileage' => $this->getFieldValue($groups, 'recharge_mileage'), // 续航里程 ], 'groups' => $groups, ]; } /** * 从车辆配置中获取能源类型 * * @param array $data JSON数据 * @return string|null 能源类型 */ public function getFuelFormByConfig(array $data): ?string { if (empty($data[0]['items']) || !is_array($data[0]['items'])) { return null; } foreach ($data[0]['items'] as $item) { if (isset($item['key']) && $item['key'] === 'fuel_form') { return $item['value'] ?? null; } } return null; } /** * 从数据中根据字段key获取值 * * @param mixed $data 数据 * @param string $field 字段名 * @return string|null */ public function getFieldValue($data, string $field): ?string { if (is_string($data)) { $data = json_decode($data, true); if (json_last_error() !== JSON_ERROR_NONE) { return null; } } if (!is_array($data)) { return null; } foreach ($data as $group) { if (isset($group['items']) && is_array($group['items'])) { foreach ($group['items'] as $item) { if (($item['key'] ?? null) === $field) { return $item['value'] ?? null; } if (!empty($item['children']) && is_array($item['children'])) { foreach ($item['children'] as $child) { if (($child['key'] ?? null) === $field) { return $child['value'] ?? null; } } } } } } return null; } /** * 从参数列表中获取指定名称的值 */ public function getParamValue(array $params, string $name): ?string { foreach ($params as $item) { if (($item['name'] ?? null) === $name) { return $item['value'] ?? null; } } return null; } /** * 判断车辆能源类型 */ public function detectEnergyType(string $text): string { if (function_exists('mb_internal_encoding')) { mb_internal_encoding('UTF-8'); } // 明确定义的新能源关键字 $newEnergyKeywords = [ '纯电', '纯电动', '电动车', 'BEV', '插电混动', '插电式混合', '插电式', '插电', 'PHEV', '增程式', '增程', 'REEV', '氢能', '氢燃料', '燃料电池', 'FCEV', '太阳能', ]; // 传统能源关键字 $traditionalKeywords = [ '汽油', '柴油', '天然气', 'LNG', 'CNG', ]; // 普通混动关键字(归为传统能源) $hybridKeywords = [ '油电混合', '油电混动', '油电', '混动', '双擎', 'HEV', ]; $contains = function (string $haystack, string $needle): bool { if ($needle === '') { return false; } if (function_exists('mb_stripos')) { return mb_stripos($haystack, $needle) !== false; } return stripos($haystack, $needle) !== false; }; foreach ($newEnergyKeywords as $kw) { if ($contains($text, $kw)) { return self::ENERGY_TYPE_RENEWABLE; } } foreach ($hybridKeywords as $kw) { if ($contains($text, $kw)) { return self::ENERGY_TYPE_CONVENTIONAL; } } foreach ($traditionalKeywords as $kw) { if ($contains($text, $kw)) { return self::ENERGY_TYPE_CONVENTIONAL; } } return self::ENERGY_TYPE_UNKNOWN; } /** * 保存二手车基本信息到数据库 * * @param array $data 数据 * @return int 插入记录ID * @throws BusinessException */ public function saveUsedCarBasicInfo(array $data): int { try { $time = time(); $data['created_at'] = $data['created_at'] ?? $time; $data['updated_at'] = $time; return (int)Db::table('usedcar_crawl_results')->insertGetId($data); } catch (Throwable $e) { throw new BusinessException('保存二手车基础信息失败:' . $e->getMessage()); } } /** * 更新配置信息到数据库 * * @param array $data 配置数据 * @param int $id 记录ID * @return bool|int * @throws BusinessException */ public function saveUsedCarConfigInfo(array $data, int $id) { try { $record = Db::table('usedcar_crawl_results')->find($id); if (!$record) { // 找不到记录就算了,不抛致命错误,视业务情况也可以抛异常 return false; } $groups = $data['groups'] ?? []; $oldFormat = []; if (!empty($record->format_car_info)) { $decoded = json_decode($record->format_car_info, true); if (json_last_error() === JSON_ERROR_NONE && is_array($decoded)) { $oldFormat = $decoded; } } $formatCarInfo = array_merge( $oldFormat, $data['format_car_info'] ?? [] ); return Db::table('usedcar_crawl_results') ->where('id', $id) ->update([ 'format_car_info' => json_encode($formatCarInfo, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), 'config_detail' => json_encode($groups, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), 'updated_at' => time(), ]); } catch (Throwable $e) { throw new BusinessException('保存车辆配置信息失败:' . $e->getMessage()); } } /** * 构建值映射表 */ public function buildValuesMap(array $carInfo): array { $valuesMap = []; // 顶层字段 foreach ($carInfo as $k => $v) { if ($k === 'info' || is_array($v)) { continue; } $valuesMap[$k] = ['value' => $v]; } // info中的配置项 if (!empty($carInfo['info']) && is_array($carInfo['info'])) { foreach ($carInfo['info'] as $k => $v) { if (is_array($v) && array_key_exists('value', $v)) { $valuesMap[$k] = $v; } } } return $valuesMap; } /** * 构建车辆规格分组 */ public function buildCarSpecGroups(array $properties, array $valuesMap): array { $groups = []; $currentKey = null; foreach ($properties as $item) { $type = (int)($item['type'] ?? 1); $key = $item['key'] ?? ''; $text = $item['text'] ?? ''; $subList = $item['sub_list'] ?? null; // 分类字段:type = 0 / 2 if (in_array($type, [0, 2], true)) { $currentKey = $key ?: ($text ?: ('group_' . count($groups))); if (!isset($groups[$currentKey])) { $groups[$currentKey] = [ 'group_key' => $currentKey, 'group_text' => $text, 'group_type' => $type, 'items' => [], ]; } continue; } // 普通字段:挂到最近的分类下 if ($currentKey === null) { $currentKey = '_default'; $groups[$currentKey] = [ 'group_key' => '_default', 'group_text' => '其他', 'group_type' => -1, 'items' => [], ]; } $value = ''; if ($key && isset($valuesMap[$key])) { $value = trim((string)($valuesMap[$key]['value'] ?? '')); } $node = [ 'key' => $key, 'text' => $text, 'value' => $value, ]; if (is_array($subList) && !empty($subList)) { $children = []; foreach ($subList as $sub) { $subKey = $sub['key'] ?? ''; $subText = $sub['text'] ?? ''; $subValue = ''; if ($subKey && isset($valuesMap[$subKey])) { $subValue = trim((string)($valuesMap[$subKey]['value'] ?? '')); } $children[] = [ 'key' => $subKey, 'text' => $subText, 'value' => $subValue, ]; } $node['children'] = $children; } $groups[$currentKey]['items'][] = $node; } return array_values($groups); } /** * 解码懂车帝价格 */ protected function decodePrice(string $str): string { $map = [ "\u{E53D}" => '1', "\u{E3F0}" => '2', "\u{E422}" => '3', "\u{E42C}" => '4', "\u{E49C}" => '5', "\u{E42B}" => '6', "\u{E4FE}" => '7', "\u{E548}" => '8', "\u{E4C8}" => '9', "\u{E453}" => '0', "\u{E45F}" => '万', ]; return strtr($str, $map); } /** * 从URL中提取ID */ protected function extractIdFromUrl(string $url) { if (preg_match('/\/(\d+)$/', $url, $matches)) { return (int)$matches[1]; } return false; } /** * 提取符合正则表达式的值(只保留数字和小数点) */ protected function extractPatternValue(string $text, string $pattern): string { $value = ''; if ($text && preg_match($pattern, $text, $m)) { $value = trim(preg_replace('/[^0-9.]/', '', $m[1])); } return $value; } /** * 下载单张图片 */ public function downloadImage(string $url, string $saveDir, ?string $filename = null, array $headers = []): ?string { if (!is_dir($saveDir)) { if (!@mkdir($saveDir, 0777, true) && !is_dir($saveDir)) { // echo "[Image] 创建目录失败:{$saveDir}\n"; return null; } } if ($filename === null) { $path = parse_url($url, PHP_URL_PATH); $ext = pathinfo($path ?? '', PATHINFO_EXTENSION); if ($ext === '') { $ext = 'jpg'; } $filename = md5($url) . '.' . $ext; } $savePath = rtrim($saveDir, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . $filename; $defaultHeaders = [ 'User-Agent: ' . self::DEFAULT_HEADERS['User-Agent'], 'Accept: image/avif,image/webp,image/apng,image/*,*/*;q=0.8', 'Referer: ' . self::DEFAULT_HEADERS['Referer'], ]; foreach ($headers as $h) { $defaultHeaders[] = $h; } $ch = curl_init($url); curl_setopt_array($ch, [ CURLOPT_RETURNTRANSFER => true, CURLOPT_FOLLOWLOCATION => true, CURLOPT_CONNECTTIMEOUT => 5, CURLOPT_TIMEOUT => 15, CURLOPT_SSL_VERIFYPEER => false, CURLOPT_SSL_VERIFYHOST => false, CURLOPT_HTTPHEADER => $defaultHeaders, ]); $data = curl_exec($ch); $errNo = curl_errno($ch); $code = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE); curl_close($ch); if ($errNo !== 0 || $code !== 200 || $data === false || $data === '') { return null; } $bytes = @file_put_contents($savePath, $data); if ($bytes === false) { return null; } return $savePath; } /** * 批量下载相册 * * @param array $urls 图片URL数组 * @param string $saveDir 保存目录 * @return array [原始URL => 保存路径或null] */ public function downloadImageAlbum(array $urls, string $saveDir): array { $result = []; foreach ($urls as $url) { $url = trim((string)$url); if ($url === '') { continue; } if (strpos($url, '//') === 0) { $url = 'https:' . $url; } try { $path = $this->downloadImage($url, $saveDir); } catch (Throwable $e) { $path = null; } $result[$url] = $path; } return $result; } /** * 下载 -> 上传 -> 删除本地 * * @param array $images 原始相册URL列表 * @param string $path 本地保存目录 * @param int $usedCarId 懂车帝 used_car_id * @param int $id 本地 crawl 记录id * @return bool */ public function uploadAndRemove(array $images, string $path, $usedCarId, $id): bool { if (empty($images)) { return false; } // 下载图片 $filePatch = $this->downloadImageAlbum($images, $path); if (empty($filePatch)) { return false; } $uploadService = new UploadService(); foreach ($filePatch as $url => $filePath) { if ($filePath === null) { continue; } try { // 上传云端 $param = [ 'path' => $filePath, 'scene' => 'usedcar_crawl', 'category' => 'usedcar', ]; $result = $uploadService->uploadLocalFile($param); if (empty($result['file_name'])) { // 上传失败则跳过,但不抛异常影响主流程 continue; } // 保存图片信息 UsedcarCrawlResultsImages::create([ 'uc_id' => $id, 'used_car_id' => $usedCarId, 'original_name'=> $result['original_name'] ?? '', 'file_path' => $result['file_name'], ]); } catch (Throwable $e) { // 单张图失败不影响其他 // 可以按需写日志 } finally { // 删除本地文件 if (is_file($filePath)) { @unlink($filePath); } } } // 删除目录 rmdir($path); return true; } /** * 安全解 JSON * * @param string $json * @param string $context * @return array * @throws BusinessException */ protected function decodeJson(string $json, string $context = 'JSON'): array { $data = json_decode($json, true); if (json_last_error() !== JSON_ERROR_NONE || !is_array($data)) { throw new BusinessException($context . ' 解析失败:' . json_last_error_msg()); } return $data; } }