From 2c295cd631a613c67f0fd012b7ad5f9953606fc0 Mon Sep 17 00:00:00 2001 From: mzl Date: Wed, 26 Nov 2025 15:51:05 +0800 Subject: [PATCH] tz --- .../UsedcarCrawlResultsController.php | 33 + .../UsedcarCrawlResultsImagesController.php | 33 + .../logic/UsedcarCrawlResultsImagesLogic.php | 17 + app/common/logic/UsedcarCrawlResultsLogic.php | 17 + app/common/model/UsedcarCrawlResults.php | 49 + .../model/UsedcarCrawlResultsImages.php | 49 + .../UsedcarCrawlResultsImagesService.php | 31 + .../service/UsedcarCrawlResultsService.php | 31 + .../utils/crawl/service/UsedCarCrawl.php | 857 ++++++++++++++++++ .../crawl/service/UsedCarCrawl_bak.php} | 17 +- .../utils/crawl/service/UsedCarCrawl_bak2.php | 798 ++++++++++++++++ .../UsedcarCrawlResultsImagesValidate.php | 29 + .../validate/UsedcarCrawlResultsValidate.php | 29 + app/controller/IndexController.php | 16 +- app/process/CheckWeb.php | 4 +- app/process/Task.php | 2 +- 16 files changed, 1995 insertions(+), 17 deletions(-) create mode 100644 app/common/controller/UsedcarCrawlResultsController.php create mode 100644 app/common/controller/UsedcarCrawlResultsImagesController.php create mode 100644 app/common/logic/UsedcarCrawlResultsImagesLogic.php create mode 100644 app/common/logic/UsedcarCrawlResultsLogic.php create mode 100644 app/common/model/UsedcarCrawlResults.php create mode 100644 app/common/model/UsedcarCrawlResultsImages.php create mode 100644 app/common/service/UsedcarCrawlResultsImagesService.php create mode 100644 app/common/service/UsedcarCrawlResultsService.php create mode 100644 app/common/utils/crawl/service/UsedCarCrawl.php rename app/common/{service/UsedCarCrawlService.php => utils/crawl/service/UsedCarCrawl_bak.php} (98%) create mode 100644 app/common/utils/crawl/service/UsedCarCrawl_bak2.php create mode 100644 app/common/validate/UsedcarCrawlResultsImagesValidate.php create mode 100644 app/common/validate/UsedcarCrawlResultsValidate.php diff --git a/app/common/controller/UsedcarCrawlResultsController.php b/app/common/controller/UsedcarCrawlResultsController.php new file mode 100644 index 0000000..bdeac97 --- /dev/null +++ b/app/common/controller/UsedcarCrawlResultsController.php @@ -0,0 +1,33 @@ +service = new UsedcarCrawlResultsService(); + } + + /** + * 列表页数据格式化 + * + * @param array $data + * @return array + */ + // protected function indexFormat($data) + // { + // if (isset($data['created_at'])) + // $data['created_at'] = date('Y-m-d H:i:s', $data['created_at']); + // if (isset($data['updated_at'])) + // $data['updated_at'] = date('Y-m-d H:i:s', $data['updated_at']); + // return $data; + // } +} \ No newline at end of file diff --git a/app/common/controller/UsedcarCrawlResultsImagesController.php b/app/common/controller/UsedcarCrawlResultsImagesController.php new file mode 100644 index 0000000..5560fda --- /dev/null +++ b/app/common/controller/UsedcarCrawlResultsImagesController.php @@ -0,0 +1,33 @@ +service = new UsedcarCrawlResultsImagesService(); + } + + /** + * 列表页数据格式化 + * + * @param array $data + * @return array + */ + // protected function indexFormat($data) + // { + // if (isset($data['created_at'])) + // $data['created_at'] = date('Y-m-d H:i:s', $data['created_at']); + // if (isset($data['updated_at'])) + // $data['updated_at'] = date('Y-m-d H:i:s', $data['updated_at']); + // return $data; + // } +} \ No newline at end of file diff --git a/app/common/logic/UsedcarCrawlResultsImagesLogic.php b/app/common/logic/UsedcarCrawlResultsImagesLogic.php new file mode 100644 index 0000000..428bc16 --- /dev/null +++ b/app/common/logic/UsedcarCrawlResultsImagesLogic.php @@ -0,0 +1,17 @@ +validator; + } +} \ No newline at end of file diff --git a/app/common/service/UsedcarCrawlResultsService.php b/app/common/service/UsedcarCrawlResultsService.php new file mode 100644 index 0000000..5aa1cdd --- /dev/null +++ b/app/common/service/UsedcarCrawlResultsService.php @@ -0,0 +1,31 @@ +validator; + } +} \ No newline at end of file diff --git a/app/common/utils/crawl/service/UsedCarCrawl.php b/app/common/utils/crawl/service/UsedCarCrawl.php new file mode 100644 index 0000000..7e99090 --- /dev/null +++ b/app/common/utils/crawl/service/UsedCarCrawl.php @@ -0,0 +1,857 @@ + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Accept-Language' => 'zh-CN,zh;q=0.9', + 'Referer' => 'https://www.dongchedi.com/', + ]; + + // Redis队列键名 + private const REDIS_QUEUE_KEY = 'usedcar:urls'; + + // 能源类型常量 + public const ENERGY_TYPE_RENEWABLE = 'RE'; + public const ENERGY_TYPE_CONVENTIONAL = 'CE'; + public const ENERGY_TYPE_UNKNOWN = 'UNKNOWN'; + + /** + * 插入队列url + * + * @param string $url URL地址 + * @param string $scene 场景：usedcar(二手信息) config(配置信息) + * @param int $id 记录ID + * @return int + */ + public function pushUrl(string $url = '', string $scene = 'usedcar', int $id = 0): int + { + $data = [ + 'scene' => $scene, + 'url' => $url, + 'id' => $id, + ]; + + return Redis::lpush( + self::REDIS_QUEUE_KEY, + json_encode($data, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) + ); + } + + /** + * 获取队列url + * + * @return array|null + * @throws BusinessException + */ + public function popUrl(): ?array + { + $data = Redis::rPop(self::REDIS_QUEUE_KEY); + if (!$data) { + return null; + } + + $task = json_decode($data, true); + if (json_last_error() !== JSON_ERROR_NONE || !is_array($task)) { + throw new BusinessException('任务数据解析失败：' . json_last_error_msg()); + } + + // 基本字段校验 + if (empty($task['url']) || empty($task['scene'])) { + throw new BusinessException('队列任务数据缺少必要字段：url 或 scene'); + } + + return $task; + } + + /** + * 执行抓取任务 + * + * @return bool + * @throws BusinessException + */ + public function execute(): bool + { + try { + $task = $this->popUrl(); + + if (!$task) { + // 队列为空，任务结束 + return false; + } + + $url = (string)$task['url']; + $scene = (string)$task['scene']; + $id = (int)($task['id'] ?? 0); + + $html = $this->fetchHtml($url); + if ($html === '') { + throw new BusinessException("页面内容为空: {$url}"); + } + + switch ($scene) { + case 'usedcar': + $info = $this->parseUsedCarDetail($html, $url); + $insertId = $this->saveUsedCarBasicInfo($info); + + // 下载图片 + 上传 + 入库（图片失败不影响主记录） + $imagesArray = json_decode($info['images'] ?? '[]', true); + if (json_last_error() !== JSON_ERROR_NONE) { + $imagesArray = []; + } + $saveDir = public_path('uploads/usedcar/' . $insertId); + $this->uploadAndRemove($imagesArray, $saveDir, $info['used_car_id'], $insertId); + + // 继续派发配置页任务 + if (!empty($info['car_id_url'])) { + $this->pushUrl($info['car_id_url'], 'config', $insertId); + } + break; + + case 'config': + if ($id <= 0) { + throw new BusinessException("配置任务缺少主记录ID"); + } + + $data = $this->parseConfigDetail($html); + $this->saveUsedCarConfigInfo($data, $id); + break; + + default: + throw new BusinessException("未知的任务场景: " . $scene); + } + + return true; + } catch (GuzzleException $e) { + throw new BusinessException("网络请求失败: " . $e->getMessage()); + } catch (BusinessException $e) { + // 业务异常原样抛出，方便上层知道是逻辑问题 + throw $e; + } catch (Throwable $th) { + // 兜底异常，避免 worker 直接挂掉 + throw new BusinessException( + "执行任务出错: " . $th->getMessage() + . ' | 行号: ' . $th->getLine() + . ' | 文件: ' . $th->getFile() + ); + } + } + + /** + * 获取网页HTML内容 + * + * @param string $url + * @return string + * @throws GuzzleException + */ + public function fetchHtml(string $url): string + { + $client = new Client(); + $response = $client->request('GET', $url, [ + 'headers' => self::DEFAULT_HEADERS, + 'timeout' => 10, + ]); + + return (string)$response->getBody(); + } + + /** + * 解析二手车详情页信息 + * + * @param string $html + * @param string $url + * @return array + * @throws BusinessException + */ + public function parseUsedCarDetail(string $html = '', string $url = ''): array + { + try { + $ql = QueryList::html($html); + } catch (Throwable $e) { + throw new BusinessException('解析 HTML 失败（详情页）: ' . $e->getMessage()); + } + + $title = trim($ql->find('h1')->text()); + $price = trim($ql->find('.head-info_price-wrap__Y4bxi > p')->text()); + $pageText = $ql->find('body')->text(); + $jsonText = $ql->find('#__NEXT_DATA__')->text(); + + if ($jsonText === '') { + throw new BusinessException('详情页未找到 __NEXT_DATA__ JSON'); + } + + $jsonArray = $this->decodeJson($jsonText, '详情页 __NEXT_DATA__'); + + // 车辆相册 + $images = $jsonArray['props']['pageProps']['skuDetail']['head_images'] ?? []; + + // 车辆ID / 年份 + $carInfo = $jsonArray['props']['pageProps']['skuDetail']['car_info'] ?? []; + $carId = $carInfo['car_id'] ?? 0; + $year = (int)($carInfo['year'] ?? 0); + + // 车辆ID URL + $carIdUrl = $carId ? ('https://www.dongchedi.com/auto/params-carIds-' . $carId) : ''; + + // 能源类型（两个字段择一） + $overview = $jsonArray['props']['pageProps']['skuDetail']['car_config_overview'] ?? []; + $fuelForm = $overview['power']['fuel_form'] ?? ''; + $newFuelForm = $overview['new_energy_power']['fuel_form'] ?? ''; + $resultFuelForm = $fuelForm === '' ? $newFuelForm : $fuelForm; + + // 使用里程 + $importantText = $jsonArray['props']['pageProps']['skuDetail']['important_text'] ?? ''; + $importantTextArray = $importantText ? explode(' | ', $importantText) : []; + $usageMileage = ''; + if (isset($importantTextArray[1])) { + $usageMileage = str_replace('万公里', '', $importantTextArray[1]); + } + + // 车辆名称 品牌、车系、车型 + $brandName = $carInfo['brand_name'] ?? ''; + $seriesName = $carInfo['series_name'] ?? ''; + $carName = $carInfo['car_name'] ?? ''; + + // 商家信息 + $shopInfo = $jsonArray['props']['pageProps']['skuDetail']['shop_info'] ?? []; + $formatShopInfo = [ + 'shop_id' => $shopInfo['shop_id'] ?? 0, + 'shop_name' => $shopInfo['shop_short_name'] ?? '', + 'shop_address' => $shopInfo['shop_address'] ?? '', + ]; + + // 简介参数 + $otherParams = $jsonArray['props']['pageProps']['skuDetail']['other_params'] ?? []; + $displacement = $this->getParamValue($otherParams, '排量'); + $gearbox = $this->getParamValue($otherParams, '变速箱'); + + $formatCarInfo = [ + 'displacement' => $displacement, + 'gearbox' => $gearbox, + ]; + + // 提取新车指导价 / 比新车省金额 + $newPrice = $this->extractPatternValue($pageText, '/新车指导价[:：]\s*([^\r\n万]*万)/u'); + $priceDifference = $this->extractPatternValue($pageText, '/比新车省[:：]\s*([^\r\n万]*万)/u'); + + // 提取车况介绍 + $conditionDesc = ''; + if ($pageText && preg_match('/车况介绍\s*(.*?)\s*推荐理由/su', $pageText, $m)) { + $conditionDesc = trim($m[1]); + } + + return [ + 'used_car_id' => $this->extractIdFromUrl($url), + 'car_id' => $carId, + 'url' => $url, + 'title' => $title, + 'price' => str_replace('万', '', $this->decodePrice($price)), + 'new_price' => str_replace('万', '', $newPrice), + 'price_difference' => str_replace('万', '', $priceDifference), + 'condition_desc' => $conditionDesc, + 'images' => json_encode($images, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), + 'car_id_url' => $carIdUrl, + 'used_detail' => $jsonText, + 'format_shop_info' => json_encode($formatShopInfo, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), + 'fuel_form' => $resultFuelForm, + 'classify_energy' => $this->detectEnergyType($resultFuelForm), + 'usage_mileage' => $usageMileage, + 'brand_name' => $brandName, + 'series_name' => $seriesName, + 'car_name' => $carName, + 'year' => $year, + 'format_car_info' => json_encode($formatCarInfo, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), + ]; + } + + /** + * 解析配置详情页信息 + * + * @param string $html + * @return array + * @throws BusinessException + */ + public function parseConfigDetail(string $html = ''): array + { + try { + $ql = QueryList::html($html); + } catch (Throwable $e) { + throw new BusinessException('解析 HTML 失败（配置页）: ' . $e->getMessage()); + } + + $jsonText = $ql->find('#__NEXT_DATA__')->text(); + if ($jsonText === '') { + throw new BusinessException('配置页未找到 __NEXT_DATA__ JSON'); + } + + $raw = $this->decodeJson($jsonText, '配置页 __NEXT_DATA__'); + + $properties = $raw['props']['pageProps']['rawData']['properties'] ?? []; + $carInfo = $raw['props']['pageProps']['rawData']['car_info'][0] ?? []; + + $valuesMap = $this->buildValuesMap($carInfo); + $groups = $this->buildCarSpecGroups($properties, $valuesMap); + + return [ + 'format_car_info' => [ + 'market_time' => $this->getFieldValue($groups, 'market_time'), // 上市时间 + 'driver_form' => $this->getFieldValue($groups, 'driver_form'), // 驱动形式 + 'seat_count' => $this->getFieldValue($groups, 'seat_count'), // 座位数 + 'engine_description' => $this->getFieldValue($groups, 'engine_description'), // 发动机 + 'engine_max_power_horsepower' => $this->getFieldValue($groups, 'engine_max_power_horsepower'), // 发动机最大功率 + 'electric_description' => $this->getFieldValue($groups, 'electric_description'), // 电动机 + 'battery_capacity' => $this->getFieldValue($groups, 'battery_capacity'), // 电池容量 + 'battery_type' => $this->getFieldValue($groups, 'battery_type'), // 电池类型 + 'recharge_mileage' => $this->getFieldValue($groups, 'recharge_mileage'), // 续航里程 + ], + 'groups' => $groups, + ]; + } + + /** + * 从车辆配置中获取能源类型 + * + * @param array $data JSON数据 + * @return string|null 能源类型 + */ + public function getFuelFormByConfig(array $data): ?string + { + if (empty($data[0]['items']) || !is_array($data[0]['items'])) { + return null; + } + + foreach ($data[0]['items'] as $item) { + if (isset($item['key']) && $item['key'] === 'fuel_form') { + return $item['value'] ?? null; + } + } + + return null; + } + + /** + * 从数据中根据字段key获取值 + * + * @param mixed $data 数据 + * @param string $field 字段名 + * @return string|null + */ + public function getFieldValue($data, string $field): ?string + { + if (is_string($data)) { + $data = json_decode($data, true); + if (json_last_error() !== JSON_ERROR_NONE) { + return null; + } + } + + if (!is_array($data)) { + return null; + } + + foreach ($data as $group) { + if (isset($group['items']) && is_array($group['items'])) { + foreach ($group['items'] as $item) { + if (($item['key'] ?? null) === $field) { + return $item['value'] ?? null; + } + + if (!empty($item['children']) && is_array($item['children'])) { + foreach ($item['children'] as $child) { + if (($child['key'] ?? null) === $field) { + return $child['value'] ?? null; + } + } + } + } + } + } + + return null; + } + + /** + * 从参数列表中获取指定名称的值 + */ + public function getParamValue(array $params, string $name): ?string + { + foreach ($params as $item) { + if (($item['name'] ?? null) === $name) { + return $item['value'] ?? null; + } + } + return null; + } + + /** + * 判断车辆能源类型 + */ + public function detectEnergyType(string $text): string + { + if (function_exists('mb_internal_encoding')) { + mb_internal_encoding('UTF-8'); + } + + // 明确定义的新能源关键字 + $newEnergyKeywords = [ + '纯电', '纯电动', '电动车', 'BEV', + '插电混动', '插电式混合', '插电式', '插电', 'PHEV', + '增程式', '增程', 'REEV', + '氢能', '氢燃料', '燃料电池', 'FCEV', + '太阳能', + ]; + + // 传统能源关键字 + $traditionalKeywords = [ + '汽油', + '柴油', + '天然气', 'LNG', 'CNG', + ]; + + // 普通混动关键字（归为传统能源） + $hybridKeywords = [ + '油电混合', '油电混动', '油电', + '混动', + '双擎', + 'HEV', + ]; + + $contains = function (string $haystack, string $needle): bool { + if ($needle === '') { + return false; + } + if (function_exists('mb_stripos')) { + return mb_stripos($haystack, $needle) !== false; + } + return stripos($haystack, $needle) !== false; + }; + + foreach ($newEnergyKeywords as $kw) { + if ($contains($text, $kw)) { + return self::ENERGY_TYPE_RENEWABLE; + } + } + + foreach ($hybridKeywords as $kw) { + if ($contains($text, $kw)) { + return self::ENERGY_TYPE_CONVENTIONAL; + } + } + + foreach ($traditionalKeywords as $kw) { + if ($contains($text, $kw)) { + return self::ENERGY_TYPE_CONVENTIONAL; + } + } + + return self::ENERGY_TYPE_UNKNOWN; + } + + /** + * 保存二手车基本信息到数据库 + * + * @param array $data 数据 + * @return int 插入记录ID + * @throws BusinessException + */ + public function saveUsedCarBasicInfo(array $data): int + { + try { + $time = time(); + $data['created_at'] = $data['created_at'] ?? $time; + $data['updated_at'] = $time; + + return (int)Db::table('usedcar_crawl_results')->insertGetId($data); + } catch (Throwable $e) { + throw new BusinessException('保存二手车基础信息失败：' . $e->getMessage()); + } + } + + /** + * 更新配置信息到数据库 + * + * @param array $data 配置数据 + * @param int $id 记录ID + * @return bool|int + * @throws BusinessException + */ + public function saveUsedCarConfigInfo(array $data, int $id) + { + try { + $record = Db::table('usedcar_crawl_results')->find($id); + + if (!$record) { + // 找不到记录就算了，不抛致命错误，视业务情况也可以抛异常 + return false; + } + + $groups = $data['groups'] ?? []; + $oldFormat = []; + + if (!empty($record->format_car_info)) { + $decoded = json_decode($record->format_car_info, true); + if (json_last_error() === JSON_ERROR_NONE && is_array($decoded)) { + $oldFormat = $decoded; + } + } + + $formatCarInfo = array_merge( + $oldFormat, + $data['format_car_info'] ?? [] + ); + + return Db::table('usedcar_crawl_results') + ->where('id', $id) + ->update([ + 'format_car_info' => json_encode($formatCarInfo, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), + 'config_detail' => json_encode($groups, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), + 'updated_at' => time(), + ]); + } catch (Throwable $e) { + throw new BusinessException('保存车辆配置信息失败：' . $e->getMessage()); + } + } + + /** + * 构建值映射表 + */ + public function buildValuesMap(array $carInfo): array + { + $valuesMap = []; + + // 顶层字段 + foreach ($carInfo as $k => $v) { + if ($k === 'info' || is_array($v)) { + continue; + } + $valuesMap[$k] = ['value' => $v]; + } + + // info中的配置项 + if (!empty($carInfo['info']) && is_array($carInfo['info'])) { + foreach ($carInfo['info'] as $k => $v) { + if (is_array($v) && array_key_exists('value', $v)) { + $valuesMap[$k] = $v; + } + } + } + + return $valuesMap; + } + + /** + * 构建车辆规格分组 + */ + public function buildCarSpecGroups(array $properties, array $valuesMap): array + { + $groups = []; + $currentKey = null; + + foreach ($properties as $item) { + $type = (int)($item['type'] ?? 1); + $key = $item['key'] ?? ''; + $text = $item['text'] ?? ''; + $subList = $item['sub_list'] ?? null; + + // 分类字段：type = 0 / 2 + if (in_array($type, [0, 2], true)) { + $currentKey = $key ?: ($text ?: ('group_' . count($groups))); + + if (!isset($groups[$currentKey])) { + $groups[$currentKey] = [ + 'group_key' => $currentKey, + 'group_text' => $text, + 'group_type' => $type, + 'items' => [], + ]; + } + + continue; + } + + // 普通字段：挂到最近的分类下 + if ($currentKey === null) { + $currentKey = '_default'; + $groups[$currentKey] = [ + 'group_key' => '_default', + 'group_text' => '其他', + 'group_type' => -1, + 'items' => [], + ]; + } + + $value = ''; + if ($key && isset($valuesMap[$key])) { + $value = trim((string)($valuesMap[$key]['value'] ?? '')); + } + + $node = [ + 'key' => $key, + 'text' => $text, + 'value' => $value, + ]; + + if (is_array($subList) && !empty($subList)) { + $children = []; + foreach ($subList as $sub) { + $subKey = $sub['key'] ?? ''; + $subText = $sub['text'] ?? ''; + $subValue = ''; + + if ($subKey && isset($valuesMap[$subKey])) { + $subValue = trim((string)($valuesMap[$subKey]['value'] ?? '')); + } + + $children[] = [ + 'key' => $subKey, + 'text' => $subText, + 'value' => $subValue, + ]; + } + + $node['children'] = $children; + } + + $groups[$currentKey]['items'][] = $node; + } + + return array_values($groups); + } + + /** + * 解码懂车帝价格 + */ + protected function decodePrice(string $str): string + { + $map = [ + "\u{E53D}" => '1', + "\u{E3F0}" => '2', + "\u{E422}" => '3', + "\u{E42C}" => '4', + "\u{E49C}" => '5', + "\u{E42B}" => '6', + "\u{E4FE}" => '7', + "\u{E548}" => '8', + "\u{E4C8}" => '9', + "\u{E453}" => '0', + "\u{E45F}" => '万', + ]; + + return strtr($str, $map); + } + + /** + * 从URL中提取ID + */ + protected function extractIdFromUrl(string $url) + { + if (preg_match('/\/(\d+)$/', $url, $matches)) { + return (int)$matches[1]; + } + return false; + } + + /** + * 提取符合正则表达式的值（只保留数字和小数点） + */ + protected function extractPatternValue(string $text, string $pattern): string + { + $value = ''; + if ($text && preg_match($pattern, $text, $m)) { + $value = trim(preg_replace('/[^0-9.]/', '', $m[1])); + } + return $value; + } + + /** + * 下载单张图片 + */ + public function downloadImage(string $url, string $saveDir, ?string $filename = null, array $headers = []): ?string + { + if (!is_dir($saveDir)) { + if (!@mkdir($saveDir, 0777, true) && !is_dir($saveDir)) { + // echo "[Image] 创建目录失败：{$saveDir}\n"; + return null; + } + } + + if ($filename === null) { + $path = parse_url($url, PHP_URL_PATH); + $ext = pathinfo($path ?? '', PATHINFO_EXTENSION); + + if ($ext === '') { + $ext = 'jpg'; + } + + $filename = md5($url) . '.' . $ext; + } + + $savePath = rtrim($saveDir, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . $filename; + + $defaultHeaders = [ + 'User-Agent: ' . self::DEFAULT_HEADERS['User-Agent'], + 'Accept: image/avif,image/webp,image/apng,image/*,*/*;q=0.8', + 'Referer: ' . self::DEFAULT_HEADERS['Referer'], + ]; + + foreach ($headers as $h) { + $defaultHeaders[] = $h; + } + + $ch = curl_init($url); + curl_setopt_array($ch, [ + CURLOPT_RETURNTRANSFER => true, + CURLOPT_FOLLOWLOCATION => true, + CURLOPT_CONNECTTIMEOUT => 5, + CURLOPT_TIMEOUT => 15, + CURLOPT_SSL_VERIFYPEER => false, + CURLOPT_SSL_VERIFYHOST => false, + CURLOPT_HTTPHEADER => $defaultHeaders, + ]); + + $data = curl_exec($ch); + $errNo = curl_errno($ch); + $code = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE); + curl_close($ch); + + if ($errNo !== 0 || $code !== 200 || $data === false || $data === '') { + return null; + } + + $bytes = @file_put_contents($savePath, $data); + if ($bytes === false) { + return null; + } + + return $savePath; + } + + /** + * 批量下载相册 + * + * @param array $urls 图片URL数组 + * @param string $saveDir 保存目录 + * @return array [原始URL => 保存路径或null] + */ + public function downloadImageAlbum(array $urls, string $saveDir): array + { + $result = []; + + foreach ($urls as $url) { + $url = trim((string)$url); + if ($url === '') { + continue; + } + + if (strpos($url, '//') === 0) { + $url = 'https:' . $url; + } + + try { + $path = $this->downloadImage($url, $saveDir); + } catch (Throwable $e) { + $path = null; + } + + $result[$url] = $path; + } + + return $result; + } + + /** + * 下载 -> 上传 -> 删除本地 + * + * @param array $images 原始相册URL列表 + * @param string $path 本地保存目录 + * @param int $usedCarId 懂车帝 used_car_id + * @param int $id 本地 crawl 记录id + * @return bool + */ + public function uploadAndRemove(array $images, string $path, $usedCarId, $id): bool + { + if (empty($images)) { + return false; + } + + // 下载图片 + $filePatch = $this->downloadImageAlbum($images, $path); + if (empty($filePatch)) { + return false; + } + + $uploadService = new UploadService(); + + foreach ($filePatch as $url => $filePath) { + if ($filePath === null) { + continue; + } + + try { + // 上传云端 + $param = [ + 'path' => $filePath, + 'scene' => 'usedcar_crawl', + 'category' => 'usedcar', + ]; + + $result = $uploadService->uploadLocalFile($param); + if (empty($result['file_name'])) { + // 上传失败则跳过，但不抛异常影响主流程 + continue; + } + + // 保存图片信息 + UsedcarCrawlResultsImages::create([ + 'uc_id' => $id, + 'used_car_id' => $usedCarId, + 'original_name'=> $result['original_name'] ?? '', + 'file_path' => $result['file_name'], + ]); + } catch (Throwable $e) { + // 单张图失败不影响其他 + // 可以按需写日志 + } finally { + // 删除本地文件 + if (is_file($filePath)) { + @unlink($filePath); + } + } + } + + // 删除目录 + rmdir($path); + + return true; + } + + /** + * 安全解 JSON + * + * @param string $json + * @param string $context + * @return array + * @throws BusinessException + */ + protected function decodeJson(string $json, string $context = 'JSON'): array + { + $data = json_decode($json, true); + if (json_last_error() !== JSON_ERROR_NONE || !is_array($data)) { + throw new BusinessException($context . ' 解析失败：' . json_last_error_msg()); + } + return $data; + } +} \ No newline at end of file diff --git a/app/common/service/UsedCarCrawlService.php b/app/common/utils/crawl/service/UsedCarCrawl_bak.php similarity index 98% rename from app/common/service/UsedCarCrawlService.php rename to app/common/utils/crawl/service/UsedCarCrawl_bak.php index c28d3a7..11ea8c8 100644 --- a/app/common/service/UsedCarCrawlService.php +++ b/app/common/utils/crawl/service/UsedCarCrawl_bak.php @@ -1,6 +1,6 @@ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Accept-Language' => 'zh-CN,zh;q=0.9', + 'Referer' => 'https://www.dongchedi.com/' + ]; + + // Redis队列键名 + private const REDIS_QUEUE_KEY = 'usedcar:urls'; + + // 能源类型常量 + public const ENERGY_TYPE_RENEWABLE = 'RE'; + public const ENERGY_TYPE_CONVENTIONAL = 'CE'; + public const ENERGY_TYPE_UNKNOWN = 'UNKNOWN'; + + /** + * 插入队列url + * + * @param string $url URL地址 + * @param string $scene 场景：usedcar(二手信息) config(配置信息) + * @param int $id 记录ID + * @return int + */ + public function pushUrl(string $url = '', string $scene = 'usedcar', int $id = 0): int + { + $data = [ + 'scene' => $scene, + 'url' => $url, + 'id' => $id + ]; + + return Redis::lpush(self::REDIS_QUEUE_KEY, json_encode($data, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES)); + } + + /** + * 获取队列url + * + * @return array|null + */ + public function popUrl(): ?array + { + $data = Redis::rPop(self::REDIS_QUEUE_KEY); + if (!$data) { + return null; + } + + return json_decode($data, true); + } + + /** + * 执行抓取任务 + * + * @return bool + * @throws BusinessException + */ + public function execute(): bool + { + try { + $task = $this->popUrl(); + + if (!$task) { + return false; + } + + $html = $this->fetchHtml($task['url']); + + switch ($task['scene']) { + case 'usedcar': + $info = $this->parseUsedCarDetail($html, $task['url']); + $id = $this->saveUsedCarBasicInfo($info); + $this->uploadAndRemove(json_decode($info['images'], true), public_path('uploads/usedcar/' . $id), $info['used_car_id'], $id); + $this->pushUrl($info['car_id_url'], 'config', $id); + break; + + case 'config': + $data = $this->parseConfigDetail($html); + $this->saveUsedCarConfigInfo($data, $task['id']); + break; + + default: + throw new BusinessException("未知的任务场景: " . $task['scene']); + } + + return true; + } catch (GuzzleException $e) { + throw new BusinessException("网络请求失败: " . $e->getMessage()); + } catch (Throwable $th) { + throw new BusinessException("执行任务出错: " . $th->getMessage() . ' | 行号: ' . $th->getLine() . ' | 文件: ' . $th->getFile()); + } + } + + /** + * 获取网页HTML内容 + * + * @param string $url + * @return string + * @throws GuzzleException + */ + public function fetchHtml(string $url): string + { + $client = new Client(); + $response = $client->request('GET', $url, [ + 'headers' => self::DEFAULT_HEADERS + ]); + return (string)$response->getBody(); + } + + /** + * 解析二手车详情页信息 + * + * @param string $html + * @param string $url + * @return array + */ + public function parseUsedCarDetail(string $html = '', string $url = ''): array + { + $ql = QueryList::html($html); + + $title = $ql->find('h1')->text(); + $price = $ql->find('.head-info_price-wrap__Y4bxi > p')->text(); + $pageText = $ql->find('body')->text(); + $jsonText = $ql->find('#__NEXT_DATA__')->text(); + + // 解析JSON数据 + $jsonArray = json_decode($jsonText, true); + if (json_last_error() !== JSON_ERROR_NONE) { + throw new BusinessException("JSON解析失败: " . json_last_error_msg()); + } + + // 车辆相册 + $images = $jsonArray['props']['pageProps']['skuDetail']['head_images'] ?? []; + + // 车辆ID + $carId = $jsonArray['props']['pageProps']['skuDetail']['car_info']['car_id'] ?? 0; + + // 车辆年份 + $year = $jsonArray['props']['pageProps']['skuDetail']['car_info']['year'] ?? 0; + + // 车辆ID URL + $carIdUrl = $carId ? ('https://www.dongchedi.com/auto/params-carIds-' . $carId) : ''; + + // 能源类型 + $fuelForm = $jsonArray['props']['pageProps']['skuDetail']['car_config_overview']['power']['fuel_form'] ?? ''; + $newFuelForm = $jsonArray['props']['pageProps']['skuDetail']['car_config_overview']['new_energy_power']['fuel_form'] ?? ''; + $resultFuelForm = $fuelForm === '' ? $newFuelForm : $fuelForm; + + // 使用里程 + $importantText = $jsonArray['props']['pageProps']['skuDetail']['important_text'] ?? ''; + $importantTextArray = explode(' | ', $importantText); + $usageMileage = str_replace('万公里', '', isset($importantTextArray[1]) ? $importantTextArray[1] : ''); + + // 车辆名称 品牌、车系、车型 + $brandName = $jsonArray['props']['pageProps']['skuDetail']['car_info']['brand_name'] ?? ''; + $seriesName = $jsonArray['props']['pageProps']['skuDetail']['car_info']['series_name'] ?? ''; + $carName = $jsonArray['props']['pageProps']['skuDetail']['car_info']['car_name'] ?? ''; + + // 商家信息 + $shopInfo = $jsonArray['props']['pageProps']['skuDetail']['shop_info'] ?? []; + $formatShopInfo = [ + 'shop_id' => $shopInfo['shop_id'] ?? 0, + 'shop_name' => $shopInfo['shop_short_name'] ?? '', + 'shop_address' => $shopInfo['shop_address'] ?? '' + ]; + + // 简介参数 + $otherParams = $jsonArray['props']['pageProps']['skuDetail']['other_params'] ?? []; + // 排量 + $displacement = $this->getParamValue($otherParams, '排量'); + // 变速箱 + $gearbox = $this->getParamValue($otherParams, '变速箱'); + // 格式化车辆信息 + $formatCarInfo = [ + 'displacement' => $displacement, + 'gearbox' => $gearbox + ]; + + // 提取新车指导价 + $newPrice = $this->extractPatternValue($pageText, '/新车指导价[:：]\s*([^\r\n万]*万)/u'); + + // 提取比新车省金额 + $priceDifference = $this->extractPatternValue($pageText, '/比新车省[:：]\s*([^\r\n万]*万)/u'); + + // 提取车况介绍 + $conditionDesc = ''; + if ($pageText) { + if (preg_match('/车况介绍\s*(.*?)\s*推荐理由/su', $pageText, $m)) { + $conditionDesc = trim($m[1]); + } + } + + // 整理车辆信息 + return [ + 'used_car_id' => $this->extractIdFromUrl($url), + 'car_id' => $carId, + 'url' => $url, + 'title' => $title, + 'price' => str_replace('万', '', $this->decodePrice($price)), + 'new_price' => str_replace('万', '', $newPrice), + 'price_difference' => str_replace('万', '', $priceDifference), + 'condition_desc' => $conditionDesc, + 'images' => json_encode($images, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), + 'car_id_url' => $carIdUrl, + 'used_detail' => $jsonText, + 'format_shop_info' => json_encode($formatShopInfo, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), + 'fuel_form' => $resultFuelForm, + 'classify_energy' => $this->detectEnergyType($resultFuelForm), + 'usage_mileage' => $usageMileage, + 'brand_name' => $brandName, + 'series_name' => $seriesName, + 'car_name' => $carName, + 'year' => $year, + 'format_car_info' => json_encode($formatCarInfo, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) + ]; + } + + /** + * 解析配置详情页信息 + * + * @param string $html + * @return array + */ + public function parseConfigDetail(string $html = ''): array + { + $ql = QueryList::html($html); + $jsonText = $ql->find('#__NEXT_DATA__')->text(); + + $raw = json_decode($jsonText, true); + if (json_last_error() !== JSON_ERROR_NONE) { + throw new BusinessException("配置页面JSON解析失败: " . json_last_error_msg()); + } + + $properties = $raw['props']['pageProps']['rawData']['properties'] ?? []; + $carInfo = $raw['props']['pageProps']['rawData']['car_info'][0] ?? []; + + $valuesMap = $this->buildValuesMap($carInfo); + $groups = $this->buildCarSpecGroups($properties, $valuesMap); + + return [ + 'format_car_info' => [ + 'market_time' => $this->getFieldValue($groups, 'market_time'), // 上市时间 + 'driver_form' => $this->getFieldValue($groups, 'driver_form'), // 驱动形式 + 'seat_count' => $this->getFieldValue($groups, 'seat_count'), // 座位数 + 'engine_description' => $this->getFieldValue($groups, 'engine_description'), // 发动机 + 'engine_max_power_horsepower' => $this->getFieldValue($groups, 'engine_max_power_horsepower'), // 发动机最大功率 + 'electric_description' => $this->getFieldValue($groups, 'electric_description'), //电动机 + 'battery_capacity' => $this->getFieldValue($groups, 'battery_capacity'), //电池容量 + 'battery_type' => $this->getFieldValue($groups, 'battery_type'), // 电池类型 + 'recharge_mileage' => $this->getFieldValue($groups, 'recharge_mileage') // 续航里程 + ], + 'groups' => $groups + ]; + } + + /** + * 从车辆配置中获取能源类型 + * + * @param array $data JSON数据 + * @return string|null 能源类型 + */ + public function getFuelFormByConfig(array $data): ?string + { + if (empty($data[0]['items'])) { + return null; + } + + foreach ($data[0]['items'] as $item) { + if (isset($item['key']) && $item['key'] === 'fuel_form') { + return $item['value'] ?? null; + } + } + + return null; + } + + /** + * 从数据中根据字段key获取值 + * + * @param mixed $data 数据 + * @param string $field 字段名 + * @return string|null + */ + public function getFieldValue($data, string $field): ?string + { + // 如果传入的是 JSON 字符串，先解码 + if (is_string($data)) { + $data = json_decode($data, true); + if (json_last_error() !== JSON_ERROR_NONE) { + return null; + } + } + + // 如果不是数组，直接返回null + if (!is_array($data)) { + return null; + } + + // 遍历寻找对应字段 + foreach ($data as $group) { + if (isset($group['items']) && is_array($group['items'])) { + foreach ($group['items'] as $item) { + if (isset($item['key']) && $item['key'] === $field) { + return $item['value'] ?? null; + } + + // 检查子元素 + if (isset($item['children']) && is_array($item['children'])) { + foreach ($item['children'] as $child) { + if (isset($child['key']) && $child['key'] === $field) { + return $child['value'] ?? null; + } + } + } + } + } + } + + return null; + } + + /** + * 从参数列表中获取指定名称的值 + * + * @param array $params 参数列表 + * @param string $name 名称 + * @return string|null + */ + public function getParamValue(array $params, string $name): ?string + { + foreach ($params as $item) { + if (isset($item['name']) && $item['name'] === $name) { + return $item['value'] ?? null; + } + } + return null; + } + + /** + * 判断车辆能源类型 + * + * @param string $text 描述文本 + * @return string 能源类型标识 + */ + public function detectEnergyType(string $text): string + { + if (function_exists('mb_internal_encoding')) { + mb_internal_encoding('UTF-8'); + } + + // 明确定义的新能源关键字 + $newEnergyKeywords = [ + // 纯电 + '纯电', '纯电动', '电动车', 'BEV', + // 插电式混合 + '插电混动', '插电式混合', '插电式', '插电', 'PHEV', + // 增程式 + '增程式', '增程', 'REEV', + // 氢能 / 燃料电池 + '氢能', '氢燃料', '燃料电池', 'FCEV', + // 其它 + '太阳能', + ]; + + // 传统能源关键字 + $traditionalKeywords = [ + '汽油', + '柴油', + '天然气', 'LNG', 'CNG', + ]; + + // 普通混动关键字（归为传统能源） + $hybridKeywords = [ + '油电混合', '油电混动', '油电', + '混动', // 不带"插电"的混动 + '双擎', // 丰田常见表达 + 'HEV', + ]; + + // 辅助函数：不区分大小写的包含判断 + $contains = function (string $haystack, string $needle): bool { + if ($needle === '') { + return false; + } + if (function_exists('mb_stripos')) { + return mb_stripos($haystack, $needle) !== false; + } + return stripos($haystack, $needle) !== false; + }; + + // 1. 先判断"明确新能源"关键词 + foreach ($newEnergyKeywords as $kw) { + if ($contains($text, $kw)) { + return self::ENERGY_TYPE_RENEWABLE; + } + } + + // 2. 再判断"普通油电混动" + foreach ($hybridKeywords as $kw) { + if ($contains($text, $kw)) { + return self::ENERGY_TYPE_CONVENTIONAL; + } + } + + // 3. 再判断传统能源 + foreach ($traditionalKeywords as $kw) { + if ($contains($text, $kw)) { + return self::ENERGY_TYPE_CONVENTIONAL; + } + } + + // 4. 都没匹配上 + return self::ENERGY_TYPE_UNKNOWN; + } + + /** + * 保存二手车基本信息到数据库 + * + * @param array $data 数据 + * @return int 插入记录ID + */ + public function saveUsedCarBasicInfo(array $data): int + { + $data['created_at'] = time(); + $data['updated_at'] = time(); + return Db::table('usedcar_crawl_results')->insertGetId($data); + } + + /** + * 更新配置信息到数据库 + * + * @param array $data 配置数据 + * @param int $id 记录ID + * @return bool|int + */ + public function saveUsedCarConfigInfo(array $data, int $id) + { + $record = Db::table('usedcar_crawl_results')->find($id); + + if (!$record) { + return false; + } + + $groups = $data['groups']; + $formatCarInfo = array_merge( + $data['format_car_info'], + json_decode($record->format_car_info, true) + ); + + return Db::table('usedcar_crawl_results') + ->where('id', $id) + ->update([ + 'format_car_info' => json_encode($formatCarInfo, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), + 'config_detail' => json_encode($groups, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), + 'updated_at' => time() + ]); + } + + /** + * 构建值映射表 + * + * @param array $carInfo 车辆信息 + * @return array + */ + public function buildValuesMap(array $carInfo): array + { + $valuesMap = []; + + // 处理顶层字段 + foreach ($carInfo as $k => $v) { + if ($k === 'info' || is_array($v)) { + continue; + } + $valuesMap[$k] = ['value' => $v]; + } + + // 处理info中的配置项 + if (!empty($carInfo['info']) && is_array($carInfo['info'])) { + foreach ($carInfo['info'] as $k => $v) { + if (is_array($v) && array_key_exists('value', $v)) { + $valuesMap[$k] = $v; + } + } + } + + return $valuesMap; + } + + /** + * 构建车辆规格分组 + * + * @param array $properties 属性列表 + * @param array $valuesMap 值映射表 + * @return array + */ + public function buildCarSpecGroups(array $properties, array $valuesMap): array + { + $groups = []; + $currentKey = null; + + foreach ($properties as $item) { + $type = (int)($item['type'] ?? 1); + $key = $item['key'] ?? ''; + $text = $item['text'] ?? ''; + $subList = $item['sub_list'] ?? null; + + // 分类字段：type = 0 / 2 + if (in_array($type, [0, 2], true)) { + $currentKey = $key ?: ($text ?: ('group_' . count($groups))); + + if (!isset($groups[$currentKey])) { + $groups[$currentKey] = [ + 'group_key' => $currentKey, + 'group_text' => $text, + 'group_type' => $type, + 'items' => [], + ]; + } + + continue; + } + + // 普通字段：挂到最近的分类下 + if ($currentKey === null) { + // 如果一开始就没有遇到分类，给一个默认分组 + $currentKey = '_default'; + $groups[$currentKey] = [ + 'group_key' => '_default', + 'group_text' => '其他', + 'group_type' => -1, + 'items' => [], + ]; + } + + // 取 value + $value = ''; + if ($key && isset($valuesMap[$key])) { + $value = trim((string)($valuesMap[$key]['value'] ?? '')); + } + + $node = [ + 'key' => $key, + 'text' => $text, + 'value' => $value, + ]; + + // 处理子项 + if (is_array($subList) && !empty($subList)) { + $children = []; + foreach ($subList as $sub) { + $subKey = $sub['key'] ?? ''; + $subText = $sub['text'] ?? ''; + $subValue = ''; + + if ($subKey && isset($valuesMap[$subKey])) { + $subValue = trim((string)($valuesMap[$subKey]['value'] ?? '')); + } + + $children[] = [ + 'key' => $subKey, + 'text' => $subText, + 'value' => $subValue, + ]; + } + + $node['children'] = $children; + } + + $groups[$currentKey]['items'][] = $node; + } + + // 转成索引数组方便前端遍历 + return array_values($groups); + } + + /** + * 解码懂车帝价格 + * + * @param string $str 加密字符串 + * @return string 解码后的字符串 + */ + protected function decodePrice(string $str): string + { + // 映射表：PUA 字符 => 数字 / 万 + $map = [ + "\u{E53D}" => '1', + "\u{E3F0}" => '2', + "\u{E422}" => '3', + "\u{E42C}" => '4', + "\u{E49C}" => '5', + "\u{E42B}" => '6', + "\u{E4FE}" => '7', + "\u{E548}" => '8', + "\u{E4C8}" => '9', + "\u{E453}" => '0', + "\u{E45F}" => '万', + ]; + + return strtr($str, $map); + } + + /** + * 从URL中提取ID + * + * @param string $url URL + * @return int|false + */ + protected function extractIdFromUrl(string $url) + { + if (preg_match('/\/(\d+)$/', $url, $matches)) { + return (int)$matches[1]; + } + return false; + } + + /** + * 提取符合正则表达式的值 + * + * @param string $text 文本 + * @param string $pattern 正则表达式 + * @return string + */ + protected function extractPatternValue(string $text, string $pattern): string + { + $value = ''; + if ($text) { + if (preg_match($pattern, $text, $m)) { + $value = trim(preg_replace('/[^0-9.]/', '', $m[1])); + } + } + return $value; + } + + /** + * 下载单张图片 + * + * @param string $url 图片完整URL + * @param string $saveDir 保存目录 + * @param string|null $filename 指定文件名 + * @param array $headers 额外HTTP头 + * @return string|null 成功返回保存路径，失败返回null + */ + public function downloadImage(string $url, string $saveDir, ?string $filename = null, array $headers = []): ?string + { + // 目录检查 & 创建 + if (!is_dir($saveDir)) { + if (!mkdir($saveDir, 0777, true) && !is_dir($saveDir)) { + echo "[Image] 创建目录失败：{$saveDir}\n"; + return null; + } + } + + // 自动生成文件名 + if ($filename === null) { + $path = parse_url($url, PHP_URL_PATH); + $ext = pathinfo($path, PATHINFO_EXTENSION); + + if ($ext === '') { + $ext = 'jpg'; + } + + $filename = md5($url) . '.' . $ext; + } + + $savePath = rtrim($saveDir, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . $filename; + + // 组装HTTP头 + $defaultHeaders = [ + 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Accept: image/avif,image/webp,image/apng,image/*,*/*;q=0.8', + 'Referer: https://www.dongchedi.com/', + ]; + + foreach ($headers as $h) { + $defaultHeaders[] = $h; + } + + // 使用cURL下载 + $ch = curl_init($url); + curl_setopt_array($ch, [ + CURLOPT_RETURNTRANSFER => true, + CURLOPT_FOLLOWLOCATION => true, + CURLOPT_CONNECTTIMEOUT => 5, + CURLOPT_TIMEOUT => 15, + CURLOPT_SSL_VERIFYPEER => false, + CURLOPT_SSL_VERIFYHOST => false, + CURLOPT_HTTPHEADER => $defaultHeaders, + ]); + + $data = curl_exec($ch); + $errNo = curl_errno($ch); + $errMsg = curl_error($ch); + $code = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE); + curl_close($ch); + + if ($errNo !== 0) { + // echo "[Image] 下载失败(cURL)：{$url} | {$errNo} {$errMsg}\n"; + return null; + } + + if ($code !== 200 || $data === false || $data === '') { + // echo "[Image] 下载失败(HTTP {$code})：{$url}\n"; + return null; + } + + // 写入本地文件 + $bytes = file_put_contents($savePath, $data); + if ($bytes === false) { + // echo "[Image] 保存失败：{$savePath}\n"; + return null; + } + + // echo "[Image] 下载成功：{$url} => {$savePath}\n"; + + return $savePath; + } + + /** + * 批量下载相册 + * + * @param array $urls 图片URL数组 + * @param string $saveDir 保存目录 + * @return array [原始URL => 保存路径或null] + */ + public function downloadImageAlbum(array $urls, string $saveDir): array + { + $result = []; + + foreach ($urls as $url) { + $url = trim($url); + if ($url === '') { + continue; + } + + // 补全协议 + if (strpos($url, '//') === 0) { + $url = 'https:' . $url; + } + + $path = $this->downloadImage($url, $saveDir); + $result[$url] = $path; + } + + return $result; + } + + + public function uploadAndRemove($images, $path, $usedCarId, $id) + { + // 下载图片 + $filePatch = $this->downloadImageAlbum($images, $path); + + // 检查图片 + if (empty($filePatch)) { + return false; + } + + // 上传图片 + foreach ($filePatch as $url => $path) { + if ($path === null) { + continue; + } + + // 上传云端 + $param = [ + 'path' => $path, + 'scene' => 'usedcar_crawl', + 'category' => 'usedcar' + ]; + $up = new \app\common\service\UploadService(); + $result = $up->uploadLocalFile($param); + + // 保存图片信息 + UsedcarCrawlResultsImages::create([ + 'uc_id' => $id, + 'used_car_id' => $usedCarId, + 'original_name' => $result['original_name'], + 'file_path' => $result['file_name'], + ]); + + // 删除本地文件 + @unlink($path); + } + } +} \ No newline at end of file diff --git a/app/common/validate/UsedcarCrawlResultsImagesValidate.php b/app/common/validate/UsedcarCrawlResultsImagesValidate.php new file mode 100644 index 0000000..e6cbbd5 --- /dev/null +++ b/app/common/validate/UsedcarCrawlResultsImagesValidate.php @@ -0,0 +1,29 @@ + [ + + ], + + 'update' => [ + + ] + ]; +} \ No newline at end of file diff --git a/app/common/validate/UsedcarCrawlResultsValidate.php b/app/common/validate/UsedcarCrawlResultsValidate.php new file mode 100644 index 0000000..f0766bd --- /dev/null +++ b/app/common/validate/UsedcarCrawlResultsValidate.php @@ -0,0 +1,29 @@ + [ + + ], + + 'update' => [ + + ] + ]; +} \ No newline at end of file diff --git a/app/controller/IndexController.php b/app/controller/IndexController.php index b27f703..4b4fb43 100644 --- a/app/controller/IndexController.php +++ b/app/controller/IndexController.php @@ -74,25 +74,25 @@ class IndexController $options = [ 'controller' => [ 'module' => 'common', - 'name' => 'finance_recon_dubai_files' + 'name' => 'usedcar_crawl_results_images' ], 'model' => [ 'module' => 'common', - 'name' => 'finance_recon_dubai_files' + 'name' => 'usedcar_crawl_results_images' ], 'service' => [ 'module' => 'common', - 'name' => 'finance_recon_dubai_files' + 'name' => 'usedcar_crawl_results_images' ], 'logic' => [ 'module' => 'common', - 'name' => 'finance_recon_dubai_files' + 'name' => 'usedcar_crawl_results_images' ], 'validate' => [ 'module' => 'common', - 'name' => 'finance_recon_dubai_files' + 'name' => 'usedcar_crawl_results_images' ], - 'table' => 'finance_recon_dubai_files' + 'table' => 'usedcar_crawl_results_images' ]; $result = (new Generate($options))->doGenerate(); @@ -159,9 +159,9 @@ class IndexController public function test(Request $request) { - $service = new \app\common\service\UsedCarCrawlService(); + $service = new \app\common\utils\crawl\service\UsedCarCrawl(); - $result = $service->iUrl('https://www.dongchedi.com/usedcar/21725410'); + $result = $service->pushUrl('https://www.dongchedi.com/usedcar/21631642'); // $result = $service->doData(); diff --git a/app/process/CheckWeb.php b/app/process/CheckWeb.php index c052352..f3439ef 100644 --- a/app/process/CheckWeb.php +++ b/app/process/CheckWeb.php @@ -24,8 +24,8 @@ class CheckWeb // 每秒执行：队列二手车数据处理 new Crontab('* * * * * *', function(){ - $service = new \app\common\service\UsedCarCrawlService(); - $service->doData(); + $service = new \app\common\utils\crawl\service\UsedCarCrawl(); + $service->execute(); }); new Crontab('*/50 * * * * *', function(){ diff --git a/app/process/Task.php b/app/process/Task.php index 7d8059c..c90ff85 100644 --- a/app/process/Task.php +++ b/app/process/Task.php @@ -15,7 +15,7 @@ class Task { // 每半小时执行一次，通知待跟进客户 new Crontab('0 */30 * * * *', function(){ - echo date('Y-m-d H:i:s') . "\n"; + // echo date('Y-m-d H:i:s') . "\n"; Log::info('send follow start',[time()]); $where = [