$scene, 'url' => $url, 'id' => $id ], JSON_UNESCAPED_UNICODE)); } /** * 获取队列url * * $scene 二手信息 usedcar 配置信息 config */ public function rUrl() { return json_decode(Redis::rPop('usedcar:urls'), true); } /** * 执行抓取 */ public function doData() { try { $raw = $this->rUrl(); if (!$raw) { return false; } $html = $this->getHtml($raw['url']); switch ($raw['scene']) { case 'usedcar': $info = $this->getUsedDetail($html, $raw['url']); $id = $this->saveUsedcarBasicToDb($info); $this->iUrl($info['car_id_url'], 'config', $id); break; case 'config': $data = $this->getConfigDetail($html); $this->saveUsedcarConfigToDb($data, $raw['id']); break; } return true; } catch (\Throwable $th) { throw new BusinessException($th->getMessage().'|'.$th->getLine().'|'.$th->getFile()); } } /** * 获取html */ public function getHtml($url) { $client = new Client(); $res = $client->request('GET', $url, [ 'headers' => [ 'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' . 'AppleWebKit/537.36 (KHTML, like Gecko) ' . 'Chrome/120.0.0.0 Safari/537.36', 'Accept-Language' => 'zh-CN,zh;q=0.9', 'Referer' => 'https://www.dongchedi.com/' ] ]); return (string)$res->getBody(); } /** * 获取二手信息 */ public function getUsedDetail(string $html = '', string $url = '') { $ql = QueryList::html($html); $title = $ql->find('h1')->text(); $price = $ql->find('.head-info_price-wrap__Y4bxi > p')->text(); $pageText = $ql->find('body')->text(); $jsonText = $ql->find('#__NEXT_DATA__')->text(); // 新车指导价 $newPrice = ''; if ($pageText) { if (preg_match('/新车指导价[::]\s*([^\r\n万]*万)/u', $pageText, $m)) { $newPrice = trim(preg_replace('/[^0-9.]/', '', $m[1])); } } // 比新车省 $priceDifference = ''; if ($pageText) { if (preg_match('/比新车省[::]\s*([^\r\n万]*万)/u', $pageText, $m)) { $priceDifference = trim(preg_replace('/[^0-9.]/', '', $m[1])); } } // 车况介绍 $conditionDesc = ''; if ($pageText) { if (preg_match('/车况介绍\s*(.*?)\s*推荐理由/su', $pageText, $m)) { $conditionDesc = trim($m[1]); } } $jsonArray = json_decode($jsonText, true); // 车辆相册 $images = $jsonArray['props']['pageProps']['skuDetail']['head_images'] ?? []; // 车辆ID $carId = $jsonArray['props']['pageProps']['skuDetail']['car_info']['car_id'] ?? 0; // 车辆年份 $year = $jsonArray['props']['pageProps']['skuDetail']['car_info']['year'] ?? 0; // 车辆ID URL $carIdUrl = $carId ? ('https://www.dongchedi.com/auto/params-carIds-' . $carId) : ''; // 能源类型 $fuelForm = $jsonArray['props']['pageProps']['skuDetail']['car_config_overview']['power']['fuel_form'] ?? ''; $newFuelForm = $jsonArray['props']['pageProps']['skuDetail']['car_config_overview']['new_energy_power']['fuel_form'] ?? ''; $resultFuelForm = $fuelForm === '' ? $newFuelForm : $fuelForm; // 使用里程 $importantText = $jsonArray['props']['pageProps']['skuDetail']['important_text'] ?? ''; $importantTextArray = explode(' | ', $importantText); $usageMileage = str_replace('万公里', '', isset($importantTextArray[1]) ? $importantTextArray[1] : ''); // 车辆名称 品牌、车系、车型 $brandName = $jsonArray['props']['pageProps']['skuDetail']['car_info']['brand_name'] ?? ''; $seriesName = $jsonArray['props']['pageProps']['skuDetail']['car_info']['series_name'] ?? ''; $carName = $jsonArray['props']['pageProps']['skuDetail']['car_info']['car_name'] ?? ''; // 商家信息 $shopInfo = $jsonArray['props']['pageProps']['skuDetail']['shop_info'] ?? []; $formatShopInfo = [ 'shop_id' => $shopInfo['shop_id'] ?? 0, 'shop_name' => $shopInfo['shop_short_name'] ?? '', 'shop_address' => $shopInfo['shop_address'] ?? '' ]; // 简介参数 $otherParams = $jsonArray['props']['pageProps']['skuDetail']['other_params'] ?? []; // 排量 $displacement = $this->getOtherParamValue($otherParams, '排量'); // 变速箱 $gearbox = $this->getOtherParamValue($otherParams, '变速箱'); // 格式化车辆信息 $formatCarInfo = [ 'displacement' => $displacement, 'gearbox' => $gearbox ]; // 整理车辆信息 $info['used_car_id'] = $this->getLastIdFromUrl($url); $info['car_id'] = $carId; $info['url'] = $url; $info['title'] = $title; $info['price'] = str_replace('万', '', $this->decodeDongchediPrice($price)); $info['new_price'] = str_replace('万', '', $newPrice); $info['price_difference'] = str_replace('万', '', $priceDifference); $info['condition_desc'] = $conditionDesc; $info['images'] = json_encode($images); $info['car_id_url'] = $carIdUrl; $info['used_detail'] = $jsonText; $info['format_shop_info'] = json_encode($formatShopInfo); $info['fuel_form'] = $resultFuelForm; $info['classify_energy'] = $this->detectEnergyType($resultFuelForm); $info['usage_mileage'] = $usageMileage; $info['brand_name'] = $brandName; $info['series_name'] = $seriesName; $info['car_name'] = $carName; $info['year'] = $year; $info['format_car_info'] = json_encode($formatCarInfo); return $info; } /** * 获取配置信息 */ public function getConfigDetail(string $html = '') { $ql = QueryList::html($html); $jsonText = $ql->find('#__NEXT_DATA__')->text(); $raw = json_decode($jsonText, true); $properties = $raw['props']['pageProps']['rawData']['properties'] ?? []; $carInfo = $raw['props']['pageProps']['rawData']['car_info'][0] ?? []; $valuesMap = $this->buildValuesMapFromCarInfo($carInfo); $groups = $this->buildCarSpecGroups($properties, $valuesMap); $data['format_car_info'] = [ 'market_time' => $this->getFieldValue($groups, 'market_time'), // 上市时间 'driver_form' => $this->getFieldValue($groups, 'driver_form'), // 驱动形式 'seat_count' => $this->getFieldValue($groups, 'seat_count'), // 座位数 'engine_description' => $this->getFieldValue($groups, 'engine_description'), // 发动机 'engine_max_power_horsepower' => $this->getFieldValue($groups, 'engine_max_power_horsepower'), // 发动机最大功率 'electric_description' => $this->getFieldValue($groups, 'electric_description'), //电动机 'battery_capacity' => $this->getFieldValue($groups, 'battery_capacity'), //电池容量 'battery_type' => $this->getFieldValue($groups, 'battery_type'), // 电池类型 'recharge_mileage' => $this->getFieldValue($groups, 'recharge_mileage') // 续航里程 ]; $data['groups'] = $groups; return $data; } /** * 从车辆配置 JSON 数据中获取能源类型 * * @param array $data JSON 数据 * @return string|null 能源类型,如 '油电混合'、'纯电动'、 */ public function getFuelFormByConfigInfo(array $data) { $fuelForm = null; foreach ($data[0]['items'] as $item) { if ($item['key'] === 'fuel_form') { $fuelForm = $item['value']; break; } } return $fuelForm; } /** * 从车辆配置 JSON 数据中根据字段 key 获取对应的 value * * @param string|array $data JSON 字符串 或 已解析的数组 * @param string $field 要查找的字段 key(如 'fuel_form') * @return string|null 找到则返回 value,否则返回 null */ public function getFieldValue($data, string $field) { // 如果传入的是 JSON 字符串,先解码 if (is_string($data)) { $data = json_decode($data, true); if (json_last_error() !== JSON_ERROR_NONE) { return null; // JSON 格式错误 } } // 遍历第一层(假设结构是 [ { "items": [...] } ]) foreach ($data as $group) { if (isset($group['items']) && is_array($group['items'])) { foreach ($group['items'] as $item) { if (isset($item['key']) && $item['key'] === $field) { return $item['value'] ?? null; } } } } return null; // 未找到 } /** * 从 other_params 数组中根据 name 获取对应的 value * * @param array $otherParams 格式为 [ ['name' => 'xxx', 'value' => 'yyy'], ... ] * @param string $name 要查找的字段名称(如 '上牌地') * @return string|null 找到则返回 value,否则返回 null */ public function getOtherParamValue(array $otherParams, string $name): ?string { foreach ($otherParams as $item) { if (isset($item['name']) && $item['name'] === $name) { return $item['value'] ?? null; } } return null; } /** * 根据给定字符串判断车辆能源类型 * * 返回: * - RE 新能源(纯电、插电混动、增程、氢燃料等) * - CE 传统能源(汽油、柴油、天然气、非插电混动等) * - UNKNOWN 无法判断 * * @param string $text 例如:'2022款 某某牌 2.0L 油电混合 双擎' * @return string */ public function detectEnergyType(string $text): string { if (function_exists('mb_internal_encoding')) { mb_internal_encoding('UTF-8'); } // 明确定义的新能源关键字(纯电 / 插电 / 增程 / 氢 / 太阳能 等) $newEnergyKeywords = [ // 纯电 '纯电', '纯电动', '电动车', 'BEV', // 插电式混合 '插电混动', '插电式混合', '插电式', '插电', 'PHEV', // 增程式 '增程式', '增程', 'REEV', // 氢能 / 燃料电池 '氢能', '氢燃料', '燃料电池', 'FCEV', // 其它 '太阳能', ]; // 传统能源关键字(包括燃油/燃气) $traditionalKeywords = [ '汽油', '柴油', '天然气', 'LNG', 'CNG', ]; // 普通混动关键字(按你的要求,统一归为传统能源) $hybridKeywords = [ '油电混合', '油电混动', '油电', '混动', // 不带“插电”的混动 '双擎', // 丰田常见表达 'HEV', ]; // 辅助:不区分大小写的包含判断,优先 mb_stripos $contains = function (string $haystack, string $needle): bool { if ($needle === '') { return false; } if (function_exists('mb_stripos')) { return mb_stripos($haystack, $needle) !== false; } return stripos($haystack, $needle) !== false; }; // 1. 先判断“明确新能源”关键词(插电、增程、氢、纯电等) foreach ($newEnergyKeywords as $kw) { if ($contains($text, $kw)) { return 'RE'; } } // 2. 再判断“普通油电混动”——按照你的业务,一律归为传统能源 foreach ($hybridKeywords as $kw) { if ($contains($text, $kw)) { return 'CE'; } } // 3. 再判断传统能源(纯燃油/气) foreach ($traditionalKeywords as $kw) { if ($contains($text, $kw)) { return 'CE'; } } // 4. 都没匹配上 return 'UNKNOWN'; } /** * 二手信息入库 */ public function saveUsedcarBasicToDb(array $data = []) { $data['created_at'] = time(); $data['updated_at'] = time(); return Db::table('usedcar_crawl_results')->insertGetId($data); } /** * 更新信息入库 */ public function saveUsedcarConfigToDb(array $data = [], int $id = 0) { $info = Db::table('usedcar_crawl_results')->find($id); if (!$info) { return false; } $groups = $data['groups']; $formatCarInfo = array_merge($data['format_car_info'], json_decode($info->format_car_info, true)); return Db::table('usedcar_crawl_results') ->where('id', $id) ->update([ 'format_car_info' => json_encode($formatCarInfo, JSON_UNESCAPED_UNICODE), 'config_detail' => json_encode($groups, JSON_UNESCAPED_UNICODE), 'updated_at' => time() ]); } /** * 把 car_info 扁平成一个 [key => ['value' => xxx]] 的 map * * @param array $carInfo rawData['car_info'][0] * @return array */ public function buildValuesMapFromCarInfo(array $carInfo): array { $valuesMap = []; // 1) car_info 顶层的一些字段(official_price、series_name 等) foreach ($carInfo as $k => $v) { if ($k === 'info' || is_array($v)) { continue; } $valuesMap[$k] = ['value' => $v]; } // 2) info 里的配置项 if (!empty($carInfo['info']) && is_array($carInfo['info'])) { foreach ($carInfo['info'] as $k => $v) { if (is_array($v) && array_key_exists('value', $v)) { $valuesMap[$k] = $v; } } } return $valuesMap; } /** * 根据 type 重新分组: * - type=0,2 为分组(左侧分类) * - 其它 type 都视为该分组下的字段 * * @param array $properties rawData['properties'] * @param array $valuesMap 来自 buildValuesMapFromCarInfo() * @return array */ public function buildCarSpecGroups(array $properties, array $valuesMap): array { $groups = []; $currentKey = null; // 当前分组 key foreach ($properties as $item) { $type = (int)($item['type'] ?? 1); $key = $item['key'] ?? ''; $text = $item['text'] ?? ''; $subList = $item['sub_list'] ?? null; // ---------- 1) 分类字段:type = 0 / 2 ---------- if (in_array($type, [0, 2], true)) { $currentKey = $key ?: ($text ?: ('group_' . count($groups))); if (!isset($groups[$currentKey])) { $groups[$currentKey] = [ 'group_key' => $currentKey, 'group_text' => $text, 'group_type' => $type, 'items' => [], ]; } continue; } // ---------- 2) 普通字段:挂到最近的分类下 ---------- if ($currentKey === null) { // 如果一开始就没有遇到分类,给一个默认分组 $currentKey = '_default'; $groups[$currentKey] = [ 'group_key' => '_default', 'group_text' => '其他', 'group_type' => -1, 'items' => [], ]; } // 取 value $value = ''; if ($key && isset($valuesMap[$key])) { $value = trim((string)($valuesMap[$key]['value'] ?? '')); } $node = [ 'key' => $key, 'text' => $text, 'value' => $value, ]; // ---------- 3) sub_list 子项(有就当成 children) ---------- if (is_array($subList) && !empty($subList)) { $children = []; foreach ($subList as $sub) { $subKey = $sub['key'] ?? ''; $subText = $sub['text'] ?? ''; $subValue = ''; if ($subKey && isset($valuesMap[$subKey])) { $subValue = trim((string)($valuesMap[$subKey]['value'] ?? '')); } $children[] = [ 'key' => $subKey, 'text' => $subText, 'value' => $subValue, ]; } $node['children'] = $children; } $groups[$currentKey]['items'][] = $node; } // 你可以按需返回关联数组或索引数组,这里转成索引数组方便前端遍历 return array_values($groups); } /** * 转义价格 */ protected function decodeDongchediPrice(string $str): string { // 映射表:PUA 字符 => 数字 / 万 $map = [ "\u{E53D}" => '1', "\u{E3F0}" => '2', "\u{E422}" => '3', "\u{E42C}" => '4', "\u{E49C}" => '5', "\u{E42B}" => '6', "\u{E4FE}" => '7', "\u{E548}" => '8', "\u{E4C8}" => '9', "\u{E453}" => '0', "\u{E45F}" => '万', ]; return strtr($str, $map); } /** * 获取URL最后ID */ protected function getLastIdFromUrl($url) { // 方法1:使用正则表达式提取最后的数字ID if (preg_match('/\/(\d+)$/', $url, $matches)) { return $matches[1]; } return false; } /** * 下载单张图片 * * @param string $url 图片完整 URL * @param string $saveDir 保存目录(如 /www/usedcar_images/2025-11-20) * @param string|null $filename 指定文件名(可选,不含路径),为空则自动生成 * @param array $headers 额外 HTTP 头(可选) * * @return string|null 成功返回保存路径,失败返回 null */ protected function downloadImage(string $url, string $saveDir, ?string $filename = null, array $headers = []): ?string { // 1. 目录检查 & 创建 if (!is_dir($saveDir)) { if (!mkdir($saveDir, 0777, true) && !is_dir($saveDir)) { echo "[Image] 创建目录失败:{$saveDir}\n"; return null; } } // 2. 自动生成文件名(基于 URL md5 + 原扩展名) if ($filename === null) { $path = parse_url($url, PHP_URL_PATH); // /xxx/xxx.jpg $ext = pathinfo($path, PATHINFO_EXTENSION); // jpg 或 webp 之类 if ($ext === '') { $ext = 'jpg'; // 没有扩展名时兜底 } $filename = md5($url) . '.' . $ext; } $savePath = rtrim($saveDir, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . $filename; // 3. 组装 HTTP 头 $defaultHeaders = [ 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' . 'AppleWebKit/537.36 (KHTML, like Gecko) ' . 'Chrome/120.0.0.0 Safari/537.36', 'Accept: image/avif,image/webp,image/apng,image/*,*/*;q=0.8', 'Referer: https://www.dongchedi.com/', // 很多图床需要 Referer ]; foreach ($headers as $h) { $defaultHeaders[] = $h; } // 4. 使用 cURL 下载 $ch = curl_init($url); curl_setopt_array($ch, [ CURLOPT_RETURNTRANSFER => true, CURLOPT_FOLLOWLOCATION => true, CURLOPT_CONNECTTIMEOUT => 5, CURLOPT_TIMEOUT => 15, CURLOPT_SSL_VERIFYPEER => false, CURLOPT_SSL_VERIFYHOST => false, CURLOPT_HTTPHEADER => $defaultHeaders, ]); $data = curl_exec($ch); $errNo = curl_errno($ch); $errMsg = curl_error($ch); $code = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE); curl_close($ch); if ($errNo !== 0) { echo "[Image] 下载失败(cURL):{$url} | {$errNo} {$errMsg}\n"; return null; } if ($code !== 200 || $data === false || $data === '') { echo "[Image] 下载失败(HTTP {$code}):{$url}\n"; return null; } // 5. 写入本地文件 $bytes = file_put_contents($savePath, $data); if ($bytes === false) { echo "[Image] 保存失败:{$savePath}\n"; return null; } echo "[Image] 下载成功:{$url} => {$savePath}\n"; return $savePath; } /** * 批量下载相册 * * @param array $urls 图片 URL 数组 * @param string $saveDir 保存目录 * * @return array [原始 URL => 保存路径或 null] */ protected function downloadImageAlbum(array $urls, string $saveDir): array { $result = []; foreach ($urls as $url) { $url = trim($url); if ($url === '') { continue; } // 如果存在 // 开头的相对协议,补全 if (strpos($url, '//') === 0) { $url = 'https:' . $url; } $path = $this->downloadImage($url, $saveDir); $result[$url] = $path; } return $result; } }