diff --git a/app/common/service/UsedCarCrawlService.php b/app/common/service/UsedCarCrawlService.php new file mode 100644 index 0000000..587b03e --- /dev/null +++ b/app/common/service/UsedCarCrawlService.php @@ -0,0 +1,647 @@ + $scene, + 'url' => $url, + 'id' => $id + ], JSON_UNESCAPED_UNICODE)); + } + + /** + * 获取队列url + * + * $scene 二手信息 usedcar 配置信息 config + */ + public function rUrl() + { + return json_decode(Redis::rPop('usedcar:urls'), true); + } + + /** + * 执行抓取 + */ + public function doData() + { + try { + $raw = $this->rUrl(); + + if (!$raw) { + return false; + } +echo '开始抓取~:'.date('Y-m-d H:i:s').PHP_EOL; + $html = $this->getHtml($raw['url']); + + switch ($raw['scene']) { + case 'usedcar': + $info = $this->getUsedDetail($html, $raw['url']); + $id = $this->saveUsedcarBasicToDb($info); + $this->iUrl($info['car_id_url'], 'config', $id); + + echo '一阶段 抓取成功~:'.date('Y-m-d H:i:s').PHP_EOL; + break; + + case 'config': + $data = $this->getConfigDetail($html); + $this->saveUsedcarConfigToDb($data, $raw['id']); + + echo '二阶段 抓取成功~:'.date('Y-m-d H:i:s').PHP_EOL; + break; + } + + return true; + } catch (\Throwable $th) { + throw new BusinessException($th->getMessage().'|'.$th->getLine().'|'.$th->getFile()); + } + } + + /** + * 获取html + */ + public function getHtml($url) + { + $client = new Client(); + $res = $client->request('GET', $url, [ + 'headers' => [ + 'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' + . 'AppleWebKit/537.36 (KHTML, like Gecko) ' + . 'Chrome/120.0.0.0 Safari/537.36', + 'Accept-Language' => 'zh-CN,zh;q=0.9', + 'Referer' => 'https://www.dongchedi.com/' + ] + ]); + return (string)$res->getBody(); + } + + /** + * 获取二手信息 + */ + public function getUsedDetail(string $html = '', string $url = '') + { + $ql = QueryList::html($html); + + $title = $ql->find('h1')->text(); + $price = $ql->find('.head-info_price-wrap__Y4bxi > p')->text(); + $pageText = $ql->find('body')->text(); + $jsonText = $ql->find('#__NEXT_DATA__')->text(); + + // 新车指导价 + $newPrice = ''; + if ($pageText) { + if (preg_match('/新车指导价[::]\s*([^\r\n万]*万)/u', $pageText, $m)) { + $newPrice = trim(preg_replace('/[^0-9.]/', '', $m[1])); + } + } + + // 比新车省 + $priceDifference = ''; + if ($pageText) { + if (preg_match('/比新车省[::]\s*([^\r\n万]*万)/u', $pageText, $m)) { + $priceDifference = trim(preg_replace('/[^0-9.]/', '', $m[1])); + } + } + + // 车况介绍 + $conditionDesc = ''; + if ($pageText) { + if (preg_match('/车况介绍\s*(.*?)\s*推荐理由/su', $pageText, $m)) { + $conditionDesc = trim($m[1]); + } + } + + $jsonArray = json_decode($jsonText, true); + + // 车辆相册 + $images = $jsonArray['props']['pageProps']['skuDetail']['head_images'] ?? []; + + // 车辆ID + $carId = $jsonArray['props']['pageProps']['skuDetail']['car_info']['car_id'] ?? 0; + + // 车辆年份 + $year = $jsonArray['props']['pageProps']['skuDetail']['car_info']['year'] ?? 0; + + // 车辆ID URL + $carIdUrl = $carId ? ('https://www.dongchedi.com/auto/params-carIds-' . $carId) : ''; + + // 能源类型 + $fuelForm = $jsonArray['props']['pageProps']['skuDetail']['car_config_overview']['power']['fuel_form'] ?? ''; + + // 使用里程 + $importantText = $jsonArray['props']['pageProps']['skuDetail']['important_text'] ?? ''; + $importantTextArray = explode(' | ', $importantText); + $usageMileage = str_replace('万公里', '', isset($importantTextArray[1]) ? $importantTextArray[1] : ''); + + // 车辆名称 品牌、车系、车型 + $brandName = $jsonArray['props']['pageProps']['skuDetail']['car_info']['brand_name'] ?? ''; + $seriesName = $jsonArray['props']['pageProps']['skuDetail']['car_info']['series_name'] ?? ''; + $carName = $jsonArray['props']['pageProps']['skuDetail']['car_info']['car_name'] ?? ''; + + // 商家信息 + $shopInfo = $jsonArray['props']['pageProps']['skuDetail']['shop_info'] ?? []; + $formatShopInfo = [ + 'shop_id' => $shopInfo['shop_id'] ?? 0, + 'shop_name' => $shopInfo['shop_short_name'] ?? '', + 'shop_address' => $shopInfo['shop_address'] ?? '' + ]; + + // 简介参数 + $otherParams = $jsonArray['props']['pageProps']['skuDetail']['other_params'] ?? []; + // 排量 + $displacement = $this->getOtherParamValue($otherParams, '排量'); + // 变速箱 + $gearbox = $this->getOtherParamValue($otherParams, '变速箱'); + // 格式化车辆信息 + $formatCarInfo = [ + 'displacement' => $displacement, + 'gearbox' => $gearbox + ]; + + // 整理车辆信息 + $info['used_car_id'] = $this->getLastIdFromUrl($url); + $info['car_id'] = $carId; + $info['url'] = $url; + $info['title'] = $title; + $info['price'] = str_replace('万', '', $this->decodeDongchediPrice($price)); + $info['new_price'] = str_replace('万', '', $newPrice); + $info['price_difference'] = str_replace('万', '', $priceDifference); + $info['condition_desc'] = $conditionDesc; + $info['images'] = json_encode($images); + $info['car_id_url'] = $carIdUrl; + $info['used_detail'] = $jsonText; + $info['format_shop_info'] = json_encode($formatShopInfo); + $info['fuel_form'] = $fuelForm; + $info['classify_energy'] = $this->classifyEnergy($fuelForm); + $info['usage_mileage'] = $usageMileage; + $info['brand_name'] = $brandName; + $info['series_name'] = $seriesName; + $info['car_name'] = $carName; + $info['year'] = $year; + $info['format_car_info'] = json_encode($formatCarInfo); + + return $info; + } + + /** + * 获取配置信息 + */ + public function getConfigDetail(string $html = '') + { + $ql = QueryList::html($html); + + $jsonText = $ql->find('#__NEXT_DATA__')->text(); + + $raw = json_decode($jsonText, true); + $properties = $raw['props']['pageProps']['rawData']['properties'] ?? []; + $carInfo = $raw['props']['pageProps']['rawData']['car_info'][0] ?? []; + + $valuesMap = $this->buildValuesMapFromCarInfo($carInfo); + $groups = $this->buildCarSpecGroups($properties, $valuesMap); + + $data['format_car_info'] = [ + 'market_time' => $this->getFieldValue($groups, 'market_time'), // 上市时间 + 'driver_form' => $this->getFieldValue($groups, 'driver_form'), // 驱动形式 + 'seat_count' => $this->getFieldValue($groups, 'seat_count'), // 座位数 + 'engine_description' => $this->getFieldValue($groups, 'engine_description'), // 发动机 + 'engine_max_power_horsepower' => $this->getFieldValue($groups, 'engine_max_power_horsepower'), // 发动机最大功率 + 'electric_description' => $this->getFieldValue($groups, 'electric_description'), //电动机 + 'battery_capacity' => $this->getFieldValue($groups, 'battery_capacity'), //电池容量 + 'battery_type' => $this->getFieldValue($groups, 'battery_type'), // 电池类型 + 'recharge_mileage' => $this->getFieldValue($groups, 'recharge_mileage') // 续航里程 + ]; + $data['groups'] = $groups; + return $data; + } + + + public function getFuelFormByConfigInfo(array $data) + { + $fuelForm = null; + foreach ($data[0]['items'] as $item) { + if ($item['key'] === 'fuel_form') { + $fuelForm = $item['value']; + break; + } + } + return $fuelForm; + } + + /** + * 从车辆配置 JSON 数据中根据字段 key 获取对应的 value + * + * @param string|array $data JSON 字符串 或 已解析的数组 + * @param string $field 要查找的字段 key(如 'fuel_form') + * @return string|null 找到则返回 value,否则返回 null + */ + public function getFieldValue($data, string $field) + { + // 如果传入的是 JSON 字符串,先解码 + if (is_string($data)) { + $data = json_decode($data, true); + if (json_last_error() !== JSON_ERROR_NONE) { + return null; // JSON 格式错误 + } + } + + // 遍历第一层(假设结构是 [ { "items": [...] } ]) + foreach ($data as $group) { + if (isset($group['items']) && is_array($group['items'])) { + foreach ($group['items'] as $item) { + if (isset($item['key']) && $item['key'] === $field) { + return $item['value'] ?? null; + } + } + } + } + + return null; // 未找到 + } + + /** + * 从 other_params 数组中根据 name 获取对应的 value + * + * @param array $otherParams 格式为 [ ['name' => 'xxx', 'value' => 'yyy'], ... ] + * @param string $name 要查找的字段名称(如 '上牌地') + * @return string|null 找到则返回 value,否则返回 null + */ + public function getOtherParamValue(array $otherParams, string $name): ?string + { + foreach ($otherParams as $item) { + if (isset($item['name']) && $item['name'] === $name) { + return $item['value'] ?? null; + } + } + return null; + } + + /** + * 获取能源类型 + */ + public function classifyEnergy($input) + { + $ce = ['汽油', '柴油', '天然气']; + $re = ['纯电', '电动', '氢能', '氢燃料', '太阳能', '油电混动', '混动', '插电混动', '插电式', '增程式', '增程']; + + // 标准化输入:去除空格等 + $input = trim($input); + + // 1. 精确或子串匹配(优先) + foreach ($re as $keyword) { + if (strpos($input, $keyword) !== false) { + return 'RE'; // 新能源 + } + } + foreach ($ce as $keyword) { + if (strpos($input, $keyword) !== false) { + return 'CE'; // 传统能源 + } + } + + // 2. 如果没匹配上,尝试模糊匹配(相似度 > 80%) + $bestMatch = null; + $bestScore = 0; + $bestType = null; + + // 检查 RE + foreach ($re as $keyword) { + similar_text($input, $keyword, $percent); + if ($percent > $bestScore && $percent >= 80) { + $bestScore = $percent; + $bestMatch = $keyword; + $bestType = 'RE'; + } + } + + // 检查 CE + foreach ($ce as $keyword) { + similar_text($input, $keyword, $percent); + if ($percent > $bestScore && $percent >= 80) { + $bestScore = $percent; + $bestMatch = $keyword; + $bestType = 'CE'; + } + } + + return $bestType ?? 'UNKNOWN'; // 未识别 + } + + /** + * 二手信息入库 + */ + public function saveUsedcarBasicToDb(array $data = []) + { + $data['created_at'] = time(); + $data['updated_at'] = time(); + return Db::table('usedcar_crawl_results')->insertGetId($data); + } + + /** + * 更新信息入库 + */ + public function saveUsedcarConfigToDb(array $data = [], int $id = 0) + { + $info = Db::table('usedcar_crawl_results')->find($id); + + if (!$info) { + return false; + } + + $groups = $data['groups']; + $formatCarInfo = array_merge($data['format_car_info'], json_decode($info->format_car_info, true)); + + return Db::table('usedcar_crawl_results') + ->where('id', $id) + ->update([ + 'format_car_info' => json_encode($formatCarInfo, JSON_UNESCAPED_UNICODE), + 'config_detail' => json_encode($groups, JSON_UNESCAPED_UNICODE), + 'updated_at' => time() + ]); + } + + /** + * 把 car_info 扁平成一个 [key => ['value' => xxx]] 的 map + * + * @param array $carInfo rawData['car_info'][0] + * @return array + */ + public function buildValuesMapFromCarInfo(array $carInfo): array + { + $valuesMap = []; + + // 1) car_info 顶层的一些字段(official_price、series_name 等) + foreach ($carInfo as $k => $v) { + if ($k === 'info' || is_array($v)) { + continue; + } + $valuesMap[$k] = ['value' => $v]; + } + + // 2) info 里的配置项 + if (!empty($carInfo['info']) && is_array($carInfo['info'])) { + foreach ($carInfo['info'] as $k => $v) { + if (is_array($v) && array_key_exists('value', $v)) { + $valuesMap[$k] = $v; + } + } + } + + return $valuesMap; + } + + /** + * 根据 type 重新分组: + * - type=0,2 为分组(左侧分类) + * - 其它 type 都视为该分组下的字段 + * + * @param array $properties rawData['properties'] + * @param array $valuesMap 来自 buildValuesMapFromCarInfo() + * @return array + */ + public function buildCarSpecGroups(array $properties, array $valuesMap): array + { + $groups = []; + $currentKey = null; // 当前分组 key + + foreach ($properties as $item) { + $type = (int)($item['type'] ?? 1); + $key = $item['key'] ?? ''; + $text = $item['text'] ?? ''; + $subList = $item['sub_list'] ?? null; + + // ---------- 1) 分类字段:type = 0 / 2 ---------- + if (in_array($type, [0, 2], true)) { + $currentKey = $key ?: ($text ?: ('group_' . count($groups))); + + if (!isset($groups[$currentKey])) { + $groups[$currentKey] = [ + 'group_key' => $currentKey, + 'group_text' => $text, + 'group_type' => $type, + 'items' => [], + ]; + } + + continue; + } + + // ---------- 2) 普通字段:挂到最近的分类下 ---------- + if ($currentKey === null) { + // 如果一开始就没有遇到分类,给一个默认分组 + $currentKey = '_default'; + $groups[$currentKey] = [ + 'group_key' => '_default', + 'group_text' => '其他', + 'group_type' => -1, + 'items' => [], + ]; + } + + // 取 value + $value = ''; + if ($key && isset($valuesMap[$key])) { + $value = trim((string)($valuesMap[$key]['value'] ?? '')); + } + + $node = [ + 'key' => $key, + 'text' => $text, + 'value' => $value, + ]; + + // ---------- 3) sub_list 子项(有就当成 children) ---------- + if (is_array($subList) && !empty($subList)) { + $children = []; + foreach ($subList as $sub) { + $subKey = $sub['key'] ?? ''; + $subText = $sub['text'] ?? ''; + $subValue = ''; + + if ($subKey && isset($valuesMap[$subKey])) { + $subValue = trim((string)($valuesMap[$subKey]['value'] ?? '')); + } + + $children[] = [ + 'key' => $subKey, + 'text' => $subText, + 'value' => $subValue, + ]; + } + + $node['children'] = $children; + } + + $groups[$currentKey]['items'][] = $node; + } + + // 你可以按需返回关联数组或索引数组,这里转成索引数组方便前端遍历 + return array_values($groups); + } + + /** + * 转义价格 + */ + protected function decodeDongchediPrice(string $str): string + { + // 映射表:PUA 字符 => 数字 / 万 + $map = [ + "\u{E53D}" => '1', + "\u{E3F0}" => '2', + "\u{E422}" => '3', + "\u{E42C}" => '4', + "\u{E49C}" => '5', + "\u{E42B}" => '6', + "\u{E4FE}" => '7', + "\u{E548}" => '8', + "\u{E4C8}" => '9', + "\u{E453}" => '0', + "\u{E45F}" => '万', + ]; + + return strtr($str, $map); + } + + /** + * 获取URL最后ID + */ + protected function getLastIdFromUrl($url) + { + // 方法1:使用正则表达式提取最后的数字ID + if (preg_match('/\/(\d+)$/', $url, $matches)) { + return $matches[1]; + } + return false; + } + + /** + * 下载单张图片 + * + * @param string $url 图片完整 URL + * @param string $saveDir 保存目录(如 /www/usedcar_images/2025-11-20) + * @param string|null $filename 指定文件名(可选,不含路径),为空则自动生成 + * @param array $headers 额外 HTTP 头(可选) + * + * @return string|null 成功返回保存路径,失败返回 null + */ + protected function downloadImage(string $url, string $saveDir, ?string $filename = null, array $headers = []): ?string + { + // 1. 目录检查 & 创建 + if (!is_dir($saveDir)) { + if (!mkdir($saveDir, 0777, true) && !is_dir($saveDir)) { + echo "[Image] 创建目录失败:{$saveDir}\n"; + return null; + } + } + + // 2. 自动生成文件名(基于 URL md5 + 原扩展名) + if ($filename === null) { + $path = parse_url($url, PHP_URL_PATH); // /xxx/xxx.jpg + $ext = pathinfo($path, PATHINFO_EXTENSION); // jpg 或 webp 之类 + + if ($ext === '') { + $ext = 'jpg'; // 没有扩展名时兜底 + } + + $filename = md5($url) . '.' . $ext; + } + + $savePath = rtrim($saveDir, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . $filename; + + // 3. 组装 HTTP 头 + $defaultHeaders = [ + 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' + . 'AppleWebKit/537.36 (KHTML, like Gecko) ' + . 'Chrome/120.0.0.0 Safari/537.36', + 'Accept: image/avif,image/webp,image/apng,image/*,*/*;q=0.8', + 'Referer: https://www.dongchedi.com/', // 很多图床需要 Referer + ]; + + foreach ($headers as $h) { + $defaultHeaders[] = $h; + } + + // 4. 使用 cURL 下载 + $ch = curl_init($url); + curl_setopt_array($ch, [ + CURLOPT_RETURNTRANSFER => true, + CURLOPT_FOLLOWLOCATION => true, + CURLOPT_CONNECTTIMEOUT => 5, + CURLOPT_TIMEOUT => 15, + CURLOPT_SSL_VERIFYPEER => false, + CURLOPT_SSL_VERIFYHOST => false, + CURLOPT_HTTPHEADER => $defaultHeaders, + ]); + + $data = curl_exec($ch); + $errNo = curl_errno($ch); + $errMsg = curl_error($ch); + $code = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE); + curl_close($ch); + + if ($errNo !== 0) { + echo "[Image] 下载失败(cURL):{$url} | {$errNo} {$errMsg}\n"; + return null; + } + + if ($code !== 200 || $data === false || $data === '') { + echo "[Image] 下载失败(HTTP {$code}):{$url}\n"; + return null; + } + + // 5. 写入本地文件 + $bytes = file_put_contents($savePath, $data); + if ($bytes === false) { + echo "[Image] 保存失败:{$savePath}\n"; + return null; + } + + echo "[Image] 下载成功:{$url} => {$savePath}\n"; + + return $savePath; + } + + /** + * 批量下载相册 + * + * @param array $urls 图片 URL 数组 + * @param string $saveDir 保存目录 + * + * @return array [原始 URL => 保存路径或 null] + */ + protected function downloadImageAlbum(array $urls, string $saveDir): array + { + $result = []; + + foreach ($urls as $url) { + $url = trim($url); + if ($url === '') { + continue; + } + + // 如果存在 // 开头的相对协议,补全 + if (strpos($url, '//') === 0) { + $url = 'https:' . $url; + } + + $path = $this->downloadImage($url, $saveDir); + $result[$url] = $path; + } + + return $result; + } +} diff --git a/app/controller/IndexController.php b/app/controller/IndexController.php index 1e83e3d..d424030 100644 --- a/app/controller/IndexController.php +++ b/app/controller/IndexController.php @@ -159,9 +159,16 @@ class IndexController public function test(Request $request) { - $result = [ - 'desc' => 'demo' - ]; - return json(['code' => 0, 'msg' => 'ok', 'data' => $result]); + $service = new \app\common\service\UsedCarCrawlService(); + + $result = $service->iUrl('https://www.dongchedi.com/usedcar/21557932'); + + // $result = $service->doData(); + + // $result = $service->decodeDongchediPrice('.'); + + // $result = $service->classifyEnergy('插电式混合动力'); + + return json($result); } } \ No newline at end of file diff --git a/app/process/CheckWeb.php b/app/process/CheckWeb.php index 8b99d61..c052352 100644 --- a/app/process/CheckWeb.php +++ b/app/process/CheckWeb.php @@ -22,6 +22,12 @@ class CheckWeb // }); // } + // 每秒执行:队列二手车数据处理 + new Crontab('* * * * * *', function(){ + $service = new \app\common\service\UsedCarCrawlService(); + $service->doData(); + }); + new Crontab('*/50 * * * * *', function(){ $userIds = User::where('status', 1)->pluck('id')->toArray(); foreach ($userIds as $userId) {