Merge branch '20260116'

2026-01-23 14:18:29 +08:00
parent d949fdbd02 68a27d645c
commit e98b29030b
4 changed files with 363 additions and 13 deletions
@@ -11,6 +11,7 @@ use app\common\service\UsedCarService;
 use app\common\utils\crawl\service\UsedCarCrawlSingle;
 use app\common\utils\pdf\service\MpdfDriver;
 use support\exception\BusinessException;
+use support\Log;
 use support\Request;

 class UsedCarController extends BaseController
@@ -128,6 +129,25 @@ class UsedCarController extends BaseController
         return $data;
     }

+    /**
+     * 移动端链接抓取
+     * @param Request $request
+     */
+     public function crawlMobileCar(Request $request)
+     {
+         try {
+             $urlShare = $request->get('url');
+
+             $finalUrl = (new UsedCarCrawlSingle())->getFinaUrl($urlShare);
+
+             $request->setGet('url', $finalUrl);
+             $data = $this->crawlCar($request);
+             return $data;
+         } catch (\Throwable $e) {
+             return $this->error($e->getMessage());
+         }
+     }
+
     public function crawlCar(Request $request)
     {
         try {
@@ -135,15 +155,16 @@ class UsedCarController extends BaseController
             if (empty($url)){
                 throw new BusinessException('url不能为空');
             }
-             $info = (new UsedCarCrawlSingle())->crawlCar($url);
+             //$info = (new UsedCarCrawlSingle())->crawlCar($url);
+             $info = $this->service->selectCraw($url);
             $data = [
                 'user_id' => $request->user['id'],
                 'product_name' => $info['title'],
                 'price' => $info['price']<200 ? $info['price']*10000 : $info['price'],
                 'source_place' => $info['source_place'],
-                 'displacement' => $info['capacity'], //排量
+                 'displacement' => $info['capacity']??'', //排量
                 'mileage' => $info['usage_mileage']*10000, //里程
-                 'power' => $info['horsepower'], //马力
+                 'power' => $info['horsepower']??'', //马力
                 'body_color' => $info['body_color'],
                 'images' => $info['image'],
                 'image_pull' => $info['image_pull'],
@@ -166,7 +187,8 @@ class UsedCarController extends BaseController
             if (str_contains($e->getMessage(), '404 Not Found')){
                 return $this->error('车辆已下架或找不到页面');
             }
-             return $this->error($e->getMessage());
+             Log::error('crawlCar fail',['param' => $request->get('url'),'message' => $e->getMessage().$e->getFile().$e->getLine()]);
+             return $this->error($e->getMessage().$e->getFile().$e->getLine());
         }
     }

@@ -2,10 +2,12 @@

 namespace app\common\service;

+use app\common\utils\crawl\service\UsedCarCrawlSingle;
 use app\common\validate\UsedCarValidate;
 use app\common\traits\crud\CrudServiceTrait;
 use app\common\model\UsedCar;
 use GuzzleHttp\Client;
+use GuzzleHttp\Exception\GuzzleException;
 use support\exception\BusinessException;
 use support\Log;
 use support\Request;
@@ -422,4 +424,50 @@ class UsedCarService

        return $path;
    }
+
+    /**
+     * 选择车辆抓取方式
+     * @param string $url
+     */
+    public function selectCraw(string $url){
+        $usedCarCrawl = new UsedCarCrawlSingle();
+        switch (true){
+            case str_contains($url,'dongchedi.com'): //懂车帝pc端
+                $info = $usedCarCrawl->crawlCar($url);
+                break;
+            case str_contains($url,'dcdapp.com'): //懂车帝移动端
+                $finalUrl = $usedCarCrawl->getPcUrl($url); //获取pc端页面链接
+                if(empty($finalUrl)){
+                    throw new \Exception('链接解析错误');
+                }
+                $info = $usedCarCrawl->crawlCar($finalUrl);
+                break;
+            case str_contains($url,'t.zijieimg.com'): //懂车帝移动端分享链接
+                $finalUrl = $usedCarCrawl->getFinaUrl($url); //解析抖音短链，并获取pc端页面链接
+                if(empty($finalUrl)){
+                    throw new \Exception('链接解析错误');
+                }
+                $info = $usedCarCrawl->crawlCar($finalUrl);
+                break;
+            case str_contains($url,'www.che168.com/dealer'): //汽车之家
+                $info = $usedCarCrawl->crawlChe168Car($url);
+                break;
+            case str_contains($url,'www.che168.com/home?infoid='): //汽车之家pc端链接转换
+                $url = $usedCarCrawl->getChe168UrlByPc($url);
+                $info = $usedCarCrawl->crawlChe168Car($url);
+                break;
+            case str_contains($url,'wap.che168.com/dealer'): //汽车之家移动端链接
+                $finalUrl = $usedCarCrawl->getChe168UrlByMobile($url);
+                $info = $usedCarCrawl->crawlChe168Car($finalUrl);
+                break;
+            case str_contains($url,'comm.app.autohome.com.cn'): //汽车之家移动端链接
+                $finalUrl = $usedCarCrawl->get168FinaUrl($url);
+                $info = $usedCarCrawl->crawlChe168Car($finalUrl);
+                break;
+            default:
+                Log::info('未知链接：'.$url);
+                throw new \Exception('加紧开发中');
+        }
+        return $info;
+    }
 }
@@ -36,10 +36,55 @@ class UsedCarCrawlSingle
            'headers' => self::DEFAULT_HEADERS,
            'timeout' => 10,
        ]);
-
        return (string)$response->getBody();
    }

+    /**
+     * 获取懂车帝移动端分享短链最终url
+     *
+     * @param string $url
+     * @return string
+     * @throws GuzzleException
+     */
+    public function getFinaUrl(string $url): string
+    {
+        $client   = new Client([
+            'verify' => false,
+        ]);
+        $response = $client->get($url, [
+            'allow_redirects' => ['track_redirects' => true],
+        ]);
+
+        $history = $response->getHeader('X-Guzzle-Redirect-History');
+        $finalUrl = end($history);
+
+        $pcUrl = $this->getPcUrl($finalUrl);
+        return $pcUrl;
+    }
+
+    /**
+     * 从移动端url获取sku_id
+     * @param string $url
+     * @return string|null
+     */
+    public function getPcUrl(string $url): ?string
+    {
+        $query = parse_url($url, PHP_URL_QUERY);
+        if (!$query) {
+            return null;
+        }
+
+        parse_str($query, $params);
+
+        return !empty($params['sku_id']) ? 'https://www.dongchedi.com/usedcar/'.$params['sku_id'] : null;
+    }
+
+    /**
+     * 抓取懂车帝二手车信息
+     * @param string $url
+     * @return array
+     * @throws GuzzleException
+     */
    public function crawlCar(string $url){
        $html = $this->fetchHtml($url);
        if ($html === '' || str_contains($html, '404 Not Found')) {
@@ -68,20 +113,206 @@ class UsedCarCrawlSingle
        return $info;
    }

+    /**
+     * 通过汽车之家分享链接获取pc端地址
+     *
+     * @param string $url
+     * @return string
+     * @throws GuzzleException
+     */
+    public function get168FinaUrl(string $url): string
+    {
+        $client = new Client([
+            'verify' => false,
+            'timeout' => 10,
+            'headers' => [
+                'User-Agent' => 'Mozilla/5.0 (Linux; Android 13; Mobile)'
+            ]
+        ]);
+
+        $response = $client->get($url);
+        $html = (string) $response->getBody();
+
+        preg_match(
+            '/var\s+oUrl\s*=\s*[\'"]([^\'"]+che168\.com[^\'"]+)[\'"]/',
+            $html,
+            $m
+        );
+
+        if(empty($m[1])){ //如果没有获取到移动端地址
+            throw new \Exception('168移动端地址解析失败');
+        }
+
+        $pcUrl = $this->getChe168UrlByMobile($m[1]);
+        return $pcUrl;
+    }
+
+    /**
+     * 根据移动端url获取汽车之家pc端url
+     *
+     * @param string $url
+     * @return string
+     * @throws GuzzleException
+     */
+    public function getChe168UrlByMobile(string $url): string
+    {
+
+        // 解析 URL获取车辆id
+        $parsedUrl = parse_url($url);
+        $path = $parsedUrl['path'];  // "/dealer/162138/56723550.html"
+        // 使用 pathinfo 获取文件名
+        $pathParts = explode('/', $path);
+        $filename = end($pathParts);  // "56723550.html"
+        $carId = pathinfo($filename, PATHINFO_FILENAME);  // "56723550"
+
+        $pcUrl = 'https://www.che168.com/dealer/111/' . $carId . '.html?pvareaid=0';
+        return $pcUrl;
+    }
+
+    /**
+     * 根据汽车之家pc端url转换为另一种页面的汽车之家pc端url
+     *
+     * @param string $url
+     * @return string
+     * @throws GuzzleException
+     */
+    public function getChe168UrlByPc(string $url): string
+    {
+        // 解析URL获取查询参数部分
+        $queryString = parse_url($url, PHP_URL_QUERY);
+        // 将查询参数解析为数组
+        parse_str($queryString, $params);
+
+        // 获取infoid
+        $carId = $params['infoid'] ?? null;
+
+        $pcUrl = 'https://www.che168.com/dealer/111/' . $carId . '.html?pvareaid=0';
+
+        return $pcUrl;
+    }
+
+    /**
+     * 抓取汽车之家数据
+     * @param Request $request
+     */
+    public function crawlChe168Car(string $url)
+    {
+        // 解析 URL获取车辆id
+        $parsedUrl = parse_url($url);
+        $path = $parsedUrl['path'];  // "/dealer/162138/56723550.html"
+        // 使用 pathinfo 获取文件名
+        $pathParts = explode('/', $path);
+        $filename = end($pathParts);  // "56723550.html"
+        $carId = pathinfo($filename, PATHINFO_FILENAME);  // "56723550"
+
+        // 初始化 Guzzle 客户端
+        $client = new Client([
+            'headers' => [
+                'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)',
+                'Accept'     => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+                'Referer'    => 'https://www.che168.com/',
+            ],
+            'verify' => false,
+            'timeout' => 10,
+        ]);
+
+
+        // 请求 HTML
+        $response = $client->get($url);
+        $html = (string)$response->getBody();
+        file_put_contents(runtime_path('craw.html'), $html);
+
+        // 检查是否获取到 HTML
+        if (empty($html)) {
+            throw new \Exception('无法获取页面 HTML');
+        }
+
+        // che168 页面为 gb2312，Linux 默认 UTF-8，直接解析会导致 DOM 解析异常。统一转 UTF-8 再解析。
+        $html = $this->ensureUtf8ForChe168($html);
+
+        // 使用 QueryList 解析 HTML
+        $ql = QueryList::html($html);
+
+        // 解析车辆列表，每辆车可能在 li.cards-li 或类似类名里
+        $cars = $ql->find('div#nav1 ul.basic-item-ul li')->map(function($item) {
+            $attribute = [];
+            $field = $item->find('.item-name')->text();
+            $value = $item->text();
+            $title = ['product_name' => '产品名称','plate_time' => '上牌时间', 'mileage' => '表显里程', 'capacity' => '排       量', 'source_place'=> '所  在  地', 'body_color'=>'车身颜色','condition_desc'=>'车况介绍',];
+            $title_key = array_flip($title);
+            if(in_array($field,$title)){
+                $attribute[$title_key[$field]] = str_replace($field, '', $value);
+            }
+
+            return $attribute;
+        })->all();
+
+        $cars = array_filter($cars);
+        $cars = array_merge(...$cars);
+        $cars['product_name'] = $cars['car_name'] = $ql->find('div.car-box .car-brand-name')->text();
+        $cars['price'] =  $ql->find('div.goodstartmoney')->text();
+        if(empty($cars['price'])){
+            $cars['price'] =  $ql->find('span#overlayPrice')->text();
+        }
+//        $cars['price'] = explode('万', $cars['price'])[0]??0;
+        $cars['price'] = str_replace(['¥', '￥', '万'], '', $cars['price']);
+        $cars['price'] = is_numeric($cars['price'])?$cars['price']*10000:$cars['price'];
+        //$cars['price'] =  $ql->find('#overlayPrice')->text();
+        $cars['title'] =  $cars['product_name'];
+        $cars['condition_desc'] =  $ql->find('div.leave-message-box #messageBox')->text();
+        $cars['usage_mileage'] =  str_replace('万公里', '', $cars['mileage']);
+
+        //品牌与车系
+        $links = $ql->find('.bread-crumbs a')->map(function ($a) {
+            return trim($a->text());
+        })->all();
+        $cars['brand_name'] = !empty($links[3]) ? str_replace('二手','',$links[3]): null;
+        $cars['brand_id'] = 0;
+        $cars['series_name'] = !empty($links[4]) ?str_replace('二手','',$links[4]): null;
+        $cars['series_id'] = 0;
+        $cars['car_id'] = 0;
+        $info = explode(' ',$cars['product_name']);
+        $info = array_filter($info, function($item) {
+            return str_contains($item, '款');
+        });
+        $year = end($info);
+        $cars['year'] = !empty($year) ? str_replace('款', '', $year):'';
+
+        // 图片：主用轮播 .swiper-container2，备用 #pic_li（懒加载 data-original）
+        $images = $this->collectChe168Images($ql, '.swiper-container2 .swiper-slide .jiaodianphotoclick img', false);
+        if (empty($images)) {
+            $images = $this->collectChe168Images($ql, '#pic_li img', true);
+        }
+        $images = array_values(array_unique($images));
+        var_dump($images);
+
+        if (empty($images)) {
+            Log::warning("crawlChe168Car:未找到任何车辆图片，URL: {$url}");
+        }
+
+        $saveDir = public_path('uploads/usedcar/' . $carId);
+        $cars['used_car_id'] = $carId;
+        $image = $this->uploadAndRemove($images, $saveDir, 'https://www.che168.com/');
+        $cars['image'] = $image['image'];
+        $cars['image_pull'] = $image['image_pull'];
+
+        return $cars;
+    }
+
    /**
     * 下载 -> 上传 -> 删除本地
     *
     * @param array  $images     原始相册URL列表
     * @param string $path       本地保存目录
     */
-    public function uploadAndRemove(array $images, string $path)
+    public function uploadAndRemove(array $images, string $path, string $referer = '')
    {
        if (empty($images)) {
            return false;
        }

        // 下载图片
-        $filePatch = $this->downloadImageAlbum($images, $path);
+        $filePatch = $this->downloadImageAlbum($images, $path,$referer);
        if (empty($filePatch)) {
            return false;
        }
@@ -144,7 +375,7 @@ class UsedCarCrawlSingle
     * @param string $saveDir 保存目录
     * @return array [原始URL => 保存路径或null]
     */
-    public function downloadImageAlbum(array $urls, string $saveDir): array
+    public function downloadImageAlbum(array $urls, string $saveDir, string $referer): array
    {
        $result = [];

@@ -159,7 +390,7 @@ class UsedCarCrawlSingle
            }

            try {
-                $path = $this->downloadImage($url, $saveDir);
+                $path = $this->downloadImage($url, $saveDir, $referer);
            } catch (Throwable $e) {
                $path = null;
            }
@@ -174,7 +405,7 @@ class UsedCarCrawlSingle
    /**
     * 下载单张图片
     */
-    public function downloadImage(string $url, string $saveDir, ?string $filename = null, array $headers = []): ?string
+    public function downloadImage(string $url, string $saveDir, string $referer, ?string $filename = null, array $headers = []): ?string
    {
        if (!is_dir($saveDir)) {
            if (!@mkdir($saveDir, 0777, true) && !is_dir($saveDir)) {
@@ -196,10 +427,12 @@ class UsedCarCrawlSingle

        $savePath = rtrim($saveDir, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . $filename;

+        $referer = empty($referer) ? self::DEFAULT_HEADERS['Referer'] : $referer;
        $defaultHeaders = [
            'User-Agent: ' . self::DEFAULT_HEADERS['User-Agent'],
            'Accept: image/avif,image/webp,image/apng,image/*,*/*;q=0.8',
-            'Referer: ' . self::DEFAULT_HEADERS['Referer'],
+//            'Referer: ' . self::DEFAULT_HEADERS['Referer'],
+            'Referer: ' . $referer,
        ];

        foreach ($headers as $h) {
@@ -416,6 +649,54 @@ class UsedCarCrawlSingle
        return null;
    }

+    /**
+     * 从 che168 页面上收集图片 URL
+     *
+     * @param \QL\QueryList $ql
+     * @param string        $selector           选择器
+     * @param bool          $preferDataOriginal 为 true 时优先用 data-original（#pic_li 懒加载）
+     * @return array<string>
+     */
+    protected function collectChe168Images($ql, string $selector, bool $preferDataOriginal): array
+    {
+        return $ql->find($selector)->map(function ($img) use ($preferDataOriginal) {
+            $src = $preferDataOriginal ? ($img->attr('data-original') ?: $img->attr('src')) : ($img->attr('src') ?: $img->attr('data-original'));
+            if (!$src || $src === '' || str_contains($src, 'default-che168')) {
+                return null;
+            }
+            if (str_starts_with($src, '//')) {
+                $src = 'https:' . $src;
+            }
+            return $src;
+        })->filter()->values()->all();
+    }
+
+    /**
+     * 将 che168 页面 (gb2312/gbk) 转为 UTF-8，避免 Linux 下 DOM 解析异常
+     *
+     * @param string $html
+     * @return string
+     */
+    protected function ensureUtf8ForChe168(string $html): string
+    {
+        if (!function_exists('mb_convert_encoding')) {
+            return $html;
+        }
+        $isGb = stripos($html, 'charset=gb2312') !== false || stripos($html, 'charset=gbk') !== false;
+        if (!$isGb) {
+            return $html;
+        }
+        $out = @mb_convert_encoding($html, 'UTF-8', 'GB2312');
+        if ($out === false || $out === '') {
+            $out = @mb_convert_encoding($html, 'UTF-8', 'GBK');
+        }
+        if ($out !== false && $out !== '') {
+            $html = $out;
+            $html = preg_replace('/charset\s*=\s*["\']?gb(2312|k)["\']?/i', 'charset="utf-8"', $html);
+        }
+        return $html;
+    }
+
    /**
     * 安全解 JSON
     *
@@ -68,12 +68,11 @@ Route::group('/common', function () {
        Route::any('/destroy', [app\common\controller\UsedCarController::class, 'destroy']);
        Route::any('/crawl', [app\common\controller\UsedCarController::class, 'crawlKeyword']);
        Route::any('/url_crawl', [app\common\controller\UsedCarController::class, 'crawlCar']);
+        Route::any('/mobile/url_crawl', [app\common\controller\UsedCarController::class, 'crawlMobileCar']);
    });

    Route::group('/user', function () {
        Route::any('/info', [app\common\controller\UserController::class, 'show']);
        Route::any('/edit', [app\common\controller\UserController::class, 'edit']);
    });
-
-
 });