diff --git a/app/common/controller/UsedCarController.php b/app/common/controller/UsedCarController.php index 688999d..e760a90 100644 --- a/app/common/controller/UsedCarController.php +++ b/app/common/controller/UsedCarController.php @@ -1,7 +1,6 @@ get('url'); + if (empty($url)){ + throw new BusinessException('url不能为空'); + } + $info = (new UsedCarCrawlSingle())->crawlCar($url); + $data = [ + 'user_id' => $request->user['id'], + 'product_name' => $info['title'], + 'price' => $info['price']<200 ? $info['price']*10000 : $info['price'], + 'source_place' => $info['source_place'], + 'displacement' => $info['capacity'], //排量 + 'mileage' => $info['usage_mileage']*10000, //里程 + 'power' => $info['horsepower'], //马力 + 'body_color' => $info['body_color'], + 'images' => $info['image'], + 'image_pull' => $info['image_pull'], + 'condition_desc' => $info['condition_desc'], + 'attachment' => '', + 'source_url' => $url, + 'crawl_car_id' => $info['car_id'], + 'crawl_brand_id' => $info['brand_id'], + 'crawl_series_id' => $info['series_id'], + 'used_car_id' => $info['used_car_id'], + ]; + + + return $this->success($data); + } catch (\Throwable $e) { + return $this->error($e->getMessage().'|'.$e->getLine().$e->getFile().$e->getTraceAsString()); + } + } + public function crawlKeyword(Request $request){ try { // $keyword = $request->get('keyword'); diff --git a/app/common/model/UsedCar.php b/app/common/model/UsedCar.php index ef61776..3f1081c 100644 --- a/app/common/model/UsedCar.php +++ b/app/common/model/UsedCar.php @@ -40,7 +40,7 @@ class UsedCar extends Model /** * 允许自动填充模型字段 */ - protected $fillable = ['id','code', 'user_id', 'product_name', 'vin', 'price', 'source_place', 'plate_place', 'plate_time', 'displacement','mileage','power', 'body_color', 'images', 'condition_desc', 'attachment', 'crawl_title', 'crawl_brand', 'crawl_brand_id', 'crawl_series', 'crawl_series_id', 'crawl_year', 'crawl_msrp', 'crawl_car_id', 'status', 'sort', 'created_at', 'updated_at']; + protected $fillable = ['id','code', 'user_id', 'product_name', 'vin', 'price', 'source_place', 'plate_place', 'plate_time', 'displacement','mileage','power', 'body_color', 'images', 'condition_desc', 'attachment', 'pdf_src', 'source_url', 'crawl_title', 'crawl_brand', 'crawl_brand_id', 'crawl_series', 'crawl_series_id', 'crawl_year', 'crawl_msrp', 'crawl_car_id', 'status', 'sort', 'created_at', 'updated_at']; /** * 不允许自动填充模型字段 diff --git a/app/common/utils/crawl/service/UsedCarCrawlSingle.php b/app/common/utils/crawl/service/UsedCarCrawlSingle.php new file mode 100644 index 0000000..3ad5f35 --- /dev/null +++ b/app/common/utils/crawl/service/UsedCarCrawlSingle.php @@ -0,0 +1,433 @@ + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Accept-Language' => 'zh-CN,zh;q=0.9', + 'Referer' => 'https://www.dongchedi.com/', + ]; + + /** + * 获取网页HTML内容 + * + * @param string $url + * @return string + * @throws GuzzleException + */ + public function fetchHtml(string $url): string + { + $client = new Client([ + 'verify' => false, + ]); + $response = $client->request('GET', $url, [ + 'headers' => self::DEFAULT_HEADERS, + 'timeout' => 10, + ]); + + return (string)$response->getBody(); + } + + public function crawlCar(string $url){ + $html = $this->fetchHtml($url); + if ($html === '') { + throw new BusinessException("页面内容为空: {$url}"); + } + + $info = $this->parseUsedCarDetail($html, $url); + +// $insertId = $this->saveUsedCarBasicInfo($info); + + // 下载图片 + 上传 + 入库(图片失败不影响主记录) + $imagesArray = json_decode($info['images'] ?? '[]', true); + if (json_last_error() !== JSON_ERROR_NONE) { + $imagesArray = []; + } + $saveDir = public_path('uploads/usedcar/' . $info['used_car_id']); + $image = $this->uploadAndRemove($imagesArray, $saveDir); + $info['image'] = $image['image']; + $info['image_pull'] = $image['image_pull']; + + // 继续派发配置页任务 +// if (!empty($info['car_id_url'])) { +// $this->pushUrl($info['car_id_url'], 'config', $insertId); +// } + + return $info; + } + + /** + * 下载 -> 上传 -> 删除本地 + * + * @param array $images 原始相册URL列表 + * @param string $path 本地保存目录 + */ + public function uploadAndRemove(array $images, string $path) + { + if (empty($images)) { + return false; + } + + // 下载图片 + $filePatch = $this->downloadImageAlbum($images, $path); + if (empty($filePatch)) { + return false; + } + + $uploadService = new UploadService(); + + $image = []; + $image_pull = []; + foreach ($filePatch as $url => $filePath) { + if ($filePath === null) { + continue; + } + + try { + // 上传云端 + $param = [ + 'path' => $filePath, + 'scene' => 'usedcar_crawl', + 'category' => 'usedcar', + ]; + + $result = $uploadService->uploadLocalFile($param); + if (empty($result['file_name'])) { + // 上传失败则跳过,但不抛异常影响主流程 + continue; + } + + // 保存图片信息 +// UsedcarCrawlResultsImages::create([ +// 'uc_id' => $id, +// 'used_car_id' => $usedCarId, +// 'original_name'=> $result['original_name'] ?? '', +// 'file_path' => $result['file_name'], +// ]); + $image[] = $result['file_name']; + $image_pull[] = $result['file_url']; + + } catch (Throwable $e) { + Log::error('crawlUsedCarImage'.$e->getMessage().'|'.$e->getLine().$e->getFile()); + // 单张图失败不影响其他 + // 可以按需写日志 + } finally { + // 删除本地文件 + if (is_file($filePath)) { + @unlink($filePath); + } + } + } + + // 删除目录 + rmdir($path); + + return ['image' => $image, 'image_pull' => $image_pull]; + } + + /** + * 批量下载相册 + * + * @param array $urls 图片URL数组 + * @param string $saveDir 保存目录 + * @return array [原始URL => 保存路径或null] + */ + public function downloadImageAlbum(array $urls, string $saveDir): array + { + $result = []; + + foreach ($urls as $url) { + $url = trim((string)$url); + if ($url === '') { + continue; + } + + if (strpos($url, '//') === 0) { + $url = 'https:' . $url; + } + + try { + $path = $this->downloadImage($url, $saveDir); + } catch (Throwable $e) { + $path = null; + } + + $result[$url] = $path; + } + + return $result; + } + + + /** + * 下载单张图片 + */ + public function downloadImage(string $url, string $saveDir, ?string $filename = null, array $headers = []): ?string + { + if (!is_dir($saveDir)) { + if (!@mkdir($saveDir, 0777, true) && !is_dir($saveDir)) { + // echo "[Image] 创建目录失败:{$saveDir}\n"; + return null; + } + } + + if ($filename === null) { + $path = parse_url($url, PHP_URL_PATH); + $ext = pathinfo($path ?? '', PATHINFO_EXTENSION); + + if ($ext === '') { + $ext = 'jpg'; + } + + $filename = md5($url) . '.' . $ext; + } + + $savePath = rtrim($saveDir, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . $filename; + + $defaultHeaders = [ + 'User-Agent: ' . self::DEFAULT_HEADERS['User-Agent'], + 'Accept: image/avif,image/webp,image/apng,image/*,*/*;q=0.8', + 'Referer: ' . self::DEFAULT_HEADERS['Referer'], + ]; + + foreach ($headers as $h) { + $defaultHeaders[] = $h; + } + + $ch = curl_init($url); + curl_setopt_array($ch, [ + CURLOPT_RETURNTRANSFER => true, + CURLOPT_FOLLOWLOCATION => true, + CURLOPT_CONNECTTIMEOUT => 5, + CURLOPT_TIMEOUT => 15, + CURLOPT_SSL_VERIFYPEER => false, + CURLOPT_SSL_VERIFYHOST => false, + CURLOPT_HTTPHEADER => $defaultHeaders, + ]); + + $data = curl_exec($ch); + $errNo = curl_errno($ch); + $code = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE); + curl_close($ch); + + if ($errNo !== 0 || $code !== 200 || $data === false || $data === '') { + return null; + } + + $bytes = @file_put_contents($savePath, $data); + if ($bytes === false) { + return null; + } + + return $savePath; + } + + /** + * 解析二手车详情页信息 + * + * @param string $html + * @param string $url + * @return array + * @throws BusinessException + */ + public function parseUsedCarDetail(string $html = '', string $url = ''): array + { + try { + $ql = QueryList::html($html); + } catch (Throwable $e) { + throw new BusinessException('解析 HTML 失败(详情页): ' . $e->getMessage()); + } + + $title = trim($ql->find('h1')->text()); + $price = trim($ql->find('.head-info_price-wrap__Y4bxi > p')->text()); + $pageText = $ql->find('body')->text(); + $jsonText = $ql->find('#__NEXT_DATA__')->text(); + + if ($jsonText === '') { + throw new BusinessException('详情页未找到 __NEXT_DATA__ JSON'); + } + + $jsonArray = $this->decodeJson($jsonText, '详情页 __NEXT_DATA__'); + + // 车辆相册 + $images = $jsonArray['props']['pageProps']['skuDetail']['head_images'] ?? []; + + // 车辆ID / 年份 + $carInfo = $jsonArray['props']['pageProps']['skuDetail']['car_info'] ?? []; + $carId = $carInfo['car_id'] ?? 0; + $year = (int)($carInfo['year'] ?? 0); + + // 车辆ID URL + $carIdUrl = $carId ? ('https://www.dongchedi.com/auto/params-carIds-' . $carId) : ''; + + // 能源类型(两个字段择一) + $overview = $jsonArray['props']['pageProps']['skuDetail']['car_config_overview'] ?? []; + $fuelForm = $overview['power']['fuel_form'] ?? ''; + $horsepower = $overview['power']['horsepower'] ?? ''; //马力 + $capacity = $overview['power']['capacity'] ?? ''; //排量 + $newFuelForm = $overview['new_energy_power']['fuel_form'] ?? ''; + $resultFuelForm = $fuelForm === '' ? $newFuelForm : $fuelForm; + + // 使用里程 + $importantText = $jsonArray['props']['pageProps']['skuDetail']['important_text'] ?? ''; + $importantTextArray = $importantText ? explode(' | ', $importantText) : []; + $usageMileage = ''; + if (isset($importantTextArray[1])) { + $usageMileage = str_replace('万公里', '', $importantTextArray[1]); + } + + // 车辆名称 品牌、车系、车型 + $brandName = $carInfo['brand_name'] ?? ''; + $brand_id = $carInfo['brand_id'] ?? ''; + $seriesName = $carInfo['series_name'] ?? ''; + $series_id = $carInfo['series_id'] ?? ''; + $carName = $carInfo['car_name'] ?? ''; + + $bodyColor = $carInfo['body_color'] ?? ''; //颜色 + + // 商家信息 + $shopInfo = $jsonArray['props']['pageProps']['skuDetail']['shop_info'] ?? []; + $formatShopInfo = [ + 'shop_id' => $shopInfo['shop_id'] ?? 0, + 'shop_name' => $shopInfo['shop_short_name'] ?? '', + 'shop_address' => $shopInfo['shop_address'] ?? '', + ]; + + // 简介参数 + $otherParams = $jsonArray['props']['pageProps']['skuDetail']['other_params'] ?? []; + $displacement = $this->getParamValue($otherParams, '排量'); + $gearbox = $this->getParamValue($otherParams, '变速箱'); + $sourcePlace = $this->getParamValue($otherParams, '车源地'); + + $formatCarInfo = [ + 'displacement' => $displacement, + 'gearbox' => $gearbox, + ]; + + // 提取新车指导价 / 比新车省金额 + $newPrice = $this->extractPatternValue($pageText, '/新车指导价[::]\s*([^\r\n万]*万)/u'); + $priceDifference = $this->extractPatternValue($pageText, '/比新车省[::]\s*([^\r\n万]*万)/u'); + + // 提取车况介绍 + $conditionDesc = ''; + if ($pageText && preg_match('/车况介绍\s*(.*?)\s*推荐理由/su', $pageText, $m)) { + $conditionDesc = trim($m[1]); + } + + + return [ + 'used_car_id' => $this->extractIdFromUrl($url), + 'car_id' => $carId, + 'url' => $url, + 'title' => $title, + 'price' => str_replace('万', '', $this->decodePrice($price)), + 'new_price' => str_replace('万', '', $newPrice), + 'price_difference' => str_replace('万', '', $priceDifference), + 'condition_desc' => $conditionDesc, + 'images' => json_encode($images, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), + 'car_id_url' => $carIdUrl, + 'used_detail' => $jsonText, + 'format_shop_info' => json_encode($formatShopInfo, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), + 'fuel_form' => $resultFuelForm, + 'usage_mileage' => $usageMileage, + 'brand_name' => $brandName, + 'series_name' => $seriesName, + 'car_name' => $carName, + 'year' => $year, + 'format_car_info' => json_encode($formatCarInfo, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), + 'body_color' => $bodyColor, + 'horsepower' => $horsepower, + 'capacity' => $capacity, + 'source_place' => $sourcePlace, + 'brand_id' => $brand_id, + 'series_id' => $series_id, + ]; + } + + /** + * 解码懂车帝价格 + */ + protected function decodePrice(string $str): string + { + $map = [ + "\u{E53D}" => '1', + "\u{E3F0}" => '2', + "\u{E422}" => '3', + "\u{E42C}" => '4', + "\u{E49C}" => '5', + "\u{E42B}" => '6', + "\u{E4FE}" => '7', + "\u{E548}" => '8', + "\u{E4C8}" => '9', + "\u{E453}" => '0', + "\u{E45F}" => '万', + ]; + + return strtr($str, $map); + } + + /** + * 从URL中提取ID + */ + protected function extractIdFromUrl(string $url) + { + if (preg_match('/\/(\d+)$/', $url, $matches)) { + return (int)$matches[1]; + } + return false; + } + + /** + * 提取符合正则表达式的值(只保留数字和小数点) + */ + protected function extractPatternValue(string $text, string $pattern): string + { + $value = ''; + if ($text && preg_match($pattern, $text, $m)) { + $value = trim(preg_replace('/[^0-9.]/', '', $m[1])); + } + return $value; + } + + /** + * 从参数列表中获取指定名称的值 + */ + public function getParamValue(array $params, string $name): ?string + { + foreach ($params as $item) { + if (($item['name'] ?? null) === $name) { + return $item['value'] ?? null; + } + } + return null; + } + + /** + * 安全解 JSON + * + * @param string $json + * @param string $context + * @return array + * @throws BusinessException + */ + protected function decodeJson(string $json, string $context = 'JSON'): array + { + $data = json_decode($json, true); + if (json_last_error() !== JSON_ERROR_NONE || !is_array($data)) { + throw new BusinessException($context . ' 解析失败:' . json_last_error_msg()); + } + return $data; + } +} \ No newline at end of file diff --git a/config/route.php b/config/route.php index 8af7574..f6b0373 100644 --- a/config/route.php +++ b/config/route.php @@ -67,6 +67,7 @@ Route::group('/common', function () { Route::any('/update/status', [app\common\controller\UsedCarController::class, 'status']); Route::any('/destroy', [app\common\controller\UsedCarController::class, 'destroy']); Route::any('/crawl', [app\common\controller\UsedCarController::class, 'crawlKeyword']); + Route::any('/url_crawl', [app\common\controller\UsedCarController::class, 'crawlCar']); }); Route::group('/user', function () {