useds/app/common/utils/crawl/service/UsedCarCrawlSingle.php

<?php

namespace app\common\utils\crawl\service;

use app\common\model\UsedcarCrawlResultsImages;
use app\common\service\UploadService;
use app\common\traits\crud\CrudServiceTrait;
use app\common\utils\crawl\service\UnCodeFont;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException;
use QL\QueryList;
use support\exception\BusinessException;
use support\Log;
use Throwable;

class UsedCarCrawlSingle
{
    use CrudServiceTrait;
    use UnCodeFont;

    // 默认请求头
    private const DEFAULT_HEADERS = [
        'User-Agent'      => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
        'Accept-Language' => 'zh-CN,zh;q=0.9',
        'Referer'         => 'https://www.dongchedi.com/',
    ];

    /**
     * 获取网页HTML内容
     *
     * @param string $url
     * @return string
     * @throws GuzzleException
     */
    public function fetchHtml(string $url): string
    {
        $client   = new Client([
            'verify' => false,
        ]);
        $response = $client->request('GET', $url, [
            'headers' => self::DEFAULT_HEADERS,
            'timeout' => 10,
        ]);
        return (string)$response->getBody();
    }

    /**
     * 获取懂车帝移动端分享短链最终url
     *
     * @param string $url
     * @return string
     * @throws GuzzleException
     */
    public function getFinaUrl(string $url): string
    {
        $client   = new Client([
            'verify' => false,
        ]);
        $response = $client->get($url, [
            'allow_redirects' => ['track_redirects' => true],
        ]);

        $history = $response->getHeader('X-Guzzle-Redirect-History');
        $finalUrl = end($history);

        $pcUrl = $this->getPcUrl($finalUrl);
        return $pcUrl;
    }

    /**
     * 从移动端url获取sku_id
     * @param string $url
     * @return string|null
     */
    public function getPcUrl(string $url): ?string
    {
        $query = parse_url($url, PHP_URL_QUERY);
        if (!$query) {
            return null;
        }

        parse_str($query, $params);

        return !empty($params['sku_id']) ? 'https://www.dongchedi.com/usedcar/'.$params['sku_id'] : null;
    }

    /**
     * 抓取懂车帝二手车信息
     * @param string $url
     * @return array
     * @throws GuzzleException
     */
    public function crawlCar(string $url){
        $html = $this->fetchHtml($url);
        if ($html === '' || str_contains($html, '404 Not Found')) {
            throw new BusinessException("页面内容为空: {$url}");
        }

        $info = $this->parseUsedCarDetail($html, $url);

//        $insertId = $this->saveUsedCarBasicInfo($info);

        // 下载图片 + 上传 + 入库（图片失败不影响主记录）
        $imagesArray = json_decode($info['images'] ?? '[]', true);
        if (json_last_error() !== JSON_ERROR_NONE) {
            $imagesArray = [];
        }
        $saveDir = public_path('uploads/usedcar/' . $info['used_car_id']);
        $image = $this->uploadAndRemove($imagesArray, $saveDir);
        $info['image'] = $image['image'];
        $info['image_pull'] = $image['image_pull'];

        // 继续派发配置页任务
//        if (!empty($info['car_id_url'])) {
//            $this->pushUrl($info['car_id_url'], 'config', $insertId);
//        }

        return $info;
    }

    /**
     * 通过汽车之家分享链接获取pc端地址
     *
     * @param string $url
     * @return string
     * @throws GuzzleException
     */
    public function get168FinaUrl(string $url): string
    {
        $client = new Client([
            'verify' => false,
            'timeout' => 10,
            'headers' => [
                'User-Agent' => 'Mozilla/5.0 (Linux; Android 13; Mobile)'
            ]
        ]);

        $response = $client->get($url);
        $html = (string) $response->getBody();

        preg_match(
            '/var\s+oUrl\s*=\s*[\'"]([^\'"]+che168\.com[^\'"]+)[\'"]/',
            $html,
            $m
        );

        if(empty($m[1])){ //如果没有获取到移动端地址
            throw new \Exception('168移动端地址解析失败');
        }

        $pcUrl = $this->getChe168UrlByMobile($m[1]);
        return $pcUrl;
    }

    /**
     * 根据移动端url获取汽车之家pc端url
     *
     * @param string $url
     * @return string
     * @throws GuzzleException
     */
    public function getChe168UrlByMobile(string $url): string
    {

        // 解析 URL获取车辆id
        $parsedUrl = parse_url($url);
        $path = $parsedUrl['path'];  // "/dealer/162138/56723550.html"
        // 使用 pathinfo 获取文件名
        $pathParts = explode('/', $path);
        $filename = end($pathParts);  // "56723550.html"
        $carId = pathinfo($filename, PATHINFO_FILENAME);  // "56723550"

        $pcUrl = 'https://www.che168.com/dealer/111/' . $carId . '.html?pvareaid=0';
        return $pcUrl;
    }

    /**
     * 根据汽车之家pc端url转换为另一种页面的汽车之家pc端url
     *
     * @param string $url
     * @return string
     * @throws GuzzleException
     */
    public function getChe168UrlByPc(string $url): string
    {
        // 解析URL获取查询参数部分
        $queryString = parse_url($url, PHP_URL_QUERY);
        // 将查询参数解析为数组
        parse_str($queryString, $params);

        // 获取infoid
        $carId = $params['infoid'] ?? null;

        $pcUrl = 'https://www.che168.com/dealer/111/' . $carId . '.html?pvareaid=0';

        return $pcUrl;
    }

    /**
     * 抓取汽车之家数据
     * @param Request $request
     */
    public function crawlChe168Car(string $url)
    {
        // 解析 URL获取车辆id
        $parsedUrl = parse_url($url);
        $path = $parsedUrl['path'];  // "/dealer/162138/56723550.html"
        // 使用 pathinfo 获取文件名
        $pathParts = explode('/', $path);
        $filename = end($pathParts);  // "56723550.html"
        $carId = pathinfo($filename, PATHINFO_FILENAME);  // "56723550"

        // 初始化 Guzzle 客户端
        $client = new Client([
            'headers' => [
                'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)',
                'Accept'     => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                'Referer'    => 'https://www.che168.com/',
            ],
            'verify' => false,
            'timeout' => 10,
        ]);


        // 请求 HTML
        $response = $client->get($url);
        $html = (string)$response->getBody();
        file_put_contents(runtime_path('craw.html'), $html);

        // 检查是否获取到 HTML
        if (empty($html)) {
            throw new \Exception('无法获取页面 HTML');
        }

        // che168 页面为 gb2312，Linux 默认 UTF-8，直接解析会导致 DOM 解析异常。统一转 UTF-8 再解析。
        $html = $this->ensureUtf8ForChe168($html);

        // 使用 QueryList 解析 HTML
        $ql = QueryList::html($html);

        // 解析车辆列表，每辆车可能在 li.cards-li 或类似类名里
        $cars = $ql->find('div#nav1 ul.basic-item-ul li')->map(function($item) {
            $attribute = [];
            $field = $item->find('.item-name')->text();
            $value = $item->text();
            $title = ['product_name' => '产品名称','plate_time' => '上牌时间', 'mileage' => '表显里程', 'capacity' => '排       量', 'source_place'=> '所  在  地', 'body_color'=>'车身颜色','condition_desc'=>'车况介绍',];
            $title_key = array_flip($title);
            if(in_array($field,$title)){
                $attribute[$title_key[$field]] = str_replace($field, '', $value);
            }

            return $attribute;
        })->all();

        $cars = array_filter($cars);
        $cars = array_merge(...$cars);
        $cars['product_name'] = $cars['car_name'] = $ql->find('div.car-box .car-brand-name')->text();
        $cars['price'] =  $ql->find('div.goodstartmoney')->text();
        if(empty($cars['price'])){
            $cars['price'] =  $ql->find('span#overlayPrice')->text();
        }
//        $cars['price'] = explode('万', $cars['price'])[0]??0;
        $cars['price'] = str_replace(['¥', '￥', '万'], '', $cars['price']);
        $cars['price'] = is_numeric($cars['price'])?$cars['price']*10000:$cars['price'];
        //$cars['price'] =  $ql->find('#overlayPrice')->text();
        $cars['title'] =  $cars['product_name'];
        $cars['condition_desc'] =  $ql->find('div.leave-message-box #messageBox')->text();
        $cars['usage_mileage'] =  str_replace('万公里', '', $cars['mileage']);

        //品牌与车系
        $links = $ql->find('.bread-crumbs a')->map(function ($a) {
            return trim($a->text());
        })->all();
        $cars['brand_name'] = !empty($links[3]) ? str_replace('二手','',$links[3]): null;
        $cars['brand_id'] = 0;
        $cars['series_name'] = !empty($links[4]) ?str_replace('二手','',$links[4]): null;
        $cars['series_id'] = 0;
        $cars['car_id'] = 0;
        $info = explode(' ',$cars['product_name']);
        $info = array_filter($info, function($item) {
            return str_contains($item, '款');
        });
        $year = end($info);
        $cars['year'] = !empty($year) ? str_replace('款', '', $year):'';

        // 图片：主用轮播 .swiper-container2，备用 #pic_li（懒加载 data-original）
        $images = $this->collectChe168Images($ql, '.swiper-container2 .swiper-slide .jiaodianphotoclick img', false);
        if (empty($images)) {
            $images = $this->collectChe168Images($ql, '#pic_li img', true);
        }
        $images = array_values(array_unique($images));

        if (empty($images)) {
            Log::warning("crawlChe168Car:未找到任何车辆图片，URL: {$url}");
        }

        $saveDir = public_path('uploads/usedcar/' . $carId);
        $cars['used_car_id'] = $carId;
        $image = $this->uploadAndRemove($images, $saveDir, 'https://www.che168.com/');
        $cars['image'] = $image['image'];
        $cars['image_pull'] = $image['image_pull'];

        return $cars;
    }

    /**
     * 下载 -> 上传 -> 删除本地
     *
     * @param array  $images     原始相册URL列表
     * @param string $path       本地保存目录
     */
    public function uploadAndRemove(array $images, string $path, string $referer = '')
    {
        if (empty($images)) {
            return false;
        }

        // 下载图片
        $filePatch = $this->downloadImageAlbum($images, $path,$referer);
        if (empty($filePatch)) {
            return false;
        }

        $uploadService = new UploadService();

        $image = [];
        $image_pull = [];
        foreach ($filePatch as $url => $filePath) {
            if ($filePath === null) {
                continue;
            }

            try {
                // 上传云端
                $param = [
                    'path'     => $filePath,
                    'scene'    => 'usedcar_crawl',
                    'category' => 'usedcar',
                ];

                $result = $uploadService->uploadLocalFile($param);
                if (empty($result['file_name'])) {
                    // 上传失败则跳过，但不抛异常影响主流程
                    continue;
                }

                // 保存图片信息
//                UsedcarCrawlResultsImages::create([
//                    'uc_id'        => $id,
//                    'used_car_id'  => $usedCarId,
//                    'original_name'=> $result['original_name'] ?? '',
//                    'file_path'    => $result['file_name'],
//                ]);
                $image[] = $result['file_name'];
                $image_pull[] = $result['file_url'];

            } catch (Throwable $e) {
                Log::error('crawlUsedCarImage'.$e->getMessage().'|'.$e->getLine().$e->getFile());
                // 单张图失败不影响其他
                // 可以按需写日志
            } finally {
                // 删除本地文件
                if (is_file($filePath)) {
                    @unlink($filePath);
                }
            }
        }

        // 删除目录
        rmdir($path);

        return ['image' => $image, 'image_pull' => $image_pull];
    }

    /**
     * 批量下载相册
     *
     * @param array  $urls    图片URL数组
     * @param string $saveDir 保存目录
     * @return array [原始URL => 保存路径或null]
     */
    public function downloadImageAlbum(array $urls, string $saveDir, string $referer): array
    {
        $result = [];

        foreach ($urls as $url) {
            $url = trim((string)$url);
            if ($url === '') {
                continue;
            }

            if (strpos($url, '//') === 0) {
                $url = 'https:' . $url;
            }

            try {
                $path = $this->downloadImage($url, $saveDir, $referer);
            } catch (Throwable $e) {
                $path = null;
            }

            $result[$url] = $path;
        }

        return $result;
    }


    /**
     * 下载单张图片
     */
    public function downloadImage(string $url, string $saveDir, string $referer, ?string $filename = null, array $headers = []): ?string
    {
        if (!is_dir($saveDir)) {
            if (!@mkdir($saveDir, 0777, true) && !is_dir($saveDir)) {
                // echo "[Image] 创建目录失败：{$saveDir}\n";
                return null;
            }
        }

        if ($filename === null) {
            $path = parse_url($url, PHP_URL_PATH);
            $ext  = pathinfo($path ?? '', PATHINFO_EXTENSION);

            if ($ext === '') {
                $ext = 'jpg';
            }

            $filename = md5($url) . '.' . $ext;
        }

        $savePath = rtrim($saveDir, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . $filename;

        $referer = empty($referer) ? self::DEFAULT_HEADERS['Referer'] : $referer;
        $defaultHeaders = [
            'User-Agent: ' . self::DEFAULT_HEADERS['User-Agent'],
            'Accept: image/avif,image/webp,image/apng,image/*,*/*;q=0.8',
//            'Referer: ' . self::DEFAULT_HEADERS['Referer'],
            'Referer: ' . $referer,
        ];

        foreach ($headers as $h) {
            $defaultHeaders[] = $h;
        }

        $ch = curl_init($url);
        curl_setopt_array($ch, [
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_CONNECTTIMEOUT => 5,
            CURLOPT_TIMEOUT        => 15,
            CURLOPT_SSL_VERIFYPEER => false,
            CURLOPT_SSL_VERIFYHOST => false,
            CURLOPT_HTTPHEADER     => $defaultHeaders,
        ]);

        $data  = curl_exec($ch);
        $errNo = curl_errno($ch);
        $code  = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE);
        curl_close($ch);

        if ($errNo !== 0 || $code !== 200 || $data === false || $data === '') {
            return null;
        }

        $bytes = @file_put_contents($savePath, $data);
        if ($bytes === false) {
            return null;
        }

        return $savePath;
    }

    /**
     * 解析二手车详情页信息
     *
     * @param string $html
     * @param string $url
     * @return array
     * @throws BusinessException
     */
    public function parseUsedCarDetail(string $html = '', string $url = ''): array
    {
        try {
            $ql = QueryList::html($html);
        } catch (Throwable $e) {
            throw new BusinessException('解析 HTML 失败（详情页）: ' . $e->getMessage());
        }

        $title    = trim($ql->find('h1')->text());
        $price    = trim($ql->find('.head-info_price-wrap__Y4bxi > p')->text());
        if(empty($price)){
            $price    = trim($ql->find('span.tw-text-color-red-500')->text());
        }
        $pageText = $ql->find('body')->text();
        $jsonText = $ql->find('#__NEXT_DATA__')->text();

        if ($jsonText === '') {
            throw new BusinessException('详情页未找到 __NEXT_DATA__ JSON');
        }

        $jsonArray = $this->decodeJson($jsonText, '详情页 __NEXT_DATA__');

        // 车辆相册
        $images = $jsonArray['props']['pageProps']['skuDetail']['head_images'] ?? [];

        // 车辆ID / 年份
        $carInfo = $jsonArray['props']['pageProps']['skuDetail']['car_info'] ?? [];
        $carId   = $carInfo['car_id'] ?? 0;
        $year    = (int)($carInfo['year'] ?? 0);

        // 车辆ID URL
        $carIdUrl = $carId ? ('https://www.dongchedi.com/auto/params-carIds-' . $carId) : '';

        // 能源类型（两个字段择一）
        $overview      = $jsonArray['props']['pageProps']['skuDetail']['car_config_overview'] ?? [];
        $fuelForm      = $overview['power']['fuel_form'] ?? '';
        $horsepower      = $overview['power']['horsepower'] ?? ''; //马力
        $capacity      = $overview['power']['capacity'] ?? ''; //排量
        $newFuelForm   = $overview['new_energy_power']['fuel_form'] ?? '';
        $resultFuelForm = $fuelForm === '' ? $newFuelForm : $fuelForm;

        // 使用里程
        $importantText      = $jsonArray['props']['pageProps']['skuDetail']['important_text'] ?? '';
        $importantTextArray = $importantText ? explode(' | ', $importantText) : [];
        $usageMileage       = '';
        if (isset($importantTextArray[1])) {
            $usageMileage = str_replace('万公里', '', $importantTextArray[1]);
        }

        // 车辆名称 品牌、车系、车型
        $brandName  = $carInfo['brand_name'] ?? '';
        $brand_id  = $carInfo['brand_id'] ?? '';
        $seriesName = $carInfo['series_name'] ?? '';
        $series_id  = $carInfo['series_id'] ?? '';
        $carName    = $carInfo['car_name'] ?? '';

        $bodyColor    = $carInfo['body_color'] ?? ''; //颜色

        // 商家信息
        $shopInfo = $jsonArray['props']['pageProps']['skuDetail']['shop_info'] ?? [];
        $formatShopInfo = [
            'shop_id'      => $shopInfo['shop_id'] ?? 0,
            'shop_name'    => $shopInfo['shop_short_name'] ?? '',
            'shop_address' => $shopInfo['shop_address'] ?? '',
        ];

        // 简介参数
        $otherParams = $jsonArray['props']['pageProps']['skuDetail']['other_params'] ?? [];
        $displacement = $this->getParamValue($otherParams, '排量');
        $gearbox      = $this->getParamValue($otherParams, '变速箱');
        $sourcePlace      = $this->getParamValue($otherParams, '车源地');
        $plateTime      = $this->getParamValue($otherParams, '上牌时间');

        $formatCarInfo = [
            'displacement' => $displacement,
            'gearbox'      => $gearbox,
        ];

        // 提取新车指导价 / 比新车省金额
        $newPrice        = $this->extractPatternValue($pageText, '/新车指导价[:：]\s*([^\r\n万]*万)/u');
        $priceDifference = $this->extractPatternValue($pageText, '/比新车省[:：]\s*([^\r\n万]*万)/u');

        // 提取车况介绍
        $conditionDesc = '';
        if ($pageText && preg_match('/车况介绍\s*(.*?)\s*推荐理由/su', $pageText, $m)) {
            $conditionDesc = trim($m[1]);
        }

        $price = str_replace('一口价', '', $price);
        if ($this->needFontDecode($price)) {
            $price = $this->decodeObfuscatedText($price, []);
        }
        if ($this->needFontDecode($usageMileage)) {
            $price = $this->decodeObfuscatedText($usageMileage, []);
        }
        return [
            'used_car_id'       => $this->extractIdFromUrl($url),
            'car_id'            => $carId,
            'url'               => $url,
            'title'             => $title,
            'price'             => str_replace('万', '', $this->decodePrice($price)),
            'new_price'         => str_replace('万', '', $newPrice),
            'price_difference'  => str_replace('万', '', $priceDifference),
            'condition_desc'    => $conditionDesc,
            'images'            => json_encode($images, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
            'car_id_url'        => $carIdUrl,
            'used_detail'       => $jsonText,
            'format_shop_info'  => json_encode($formatShopInfo, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
            'fuel_form'         => $resultFuelForm,
            'usage_mileage'     => $usageMileage,
            'brand_name'        => $brandName,
            'series_name'       => $seriesName,
            'car_name'          => $carName,
            'year'              => $year,
            'format_car_info'   => json_encode($formatCarInfo, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
            'body_color'              => $bodyColor,
            'horsepower'              => $horsepower,
            'capacity'              => $capacity,
            'source_place'              => $sourcePlace,
            'brand_id'              => $brand_id,
            'series_id'              => $series_id,
            'plate_time'              => $plateTime,
        ];
    }

    /**
     * 解码懂车帝价格
     */
    protected function decodePrice(string $str): string
    {
        $map = [
            "\u{E53D}" => '1',
            "\u{E3F0}" => '2',
            "\u{E422}" => '3',
            "\u{E42C}" => '4',
            "\u{E49C}" => '5',
            "\u{E42B}" => '6',
            "\u{E4FE}" => '7',
            "\u{E548}" => '8',
            "\u{E4C8}" => '9',
            "\u{E453}" => '0',
            "\u{E45F}" => '万',
        ];

        return strtr($str, $map);
    }

    /**
     * 从URL中提取ID
     */
    protected function extractIdFromUrl(string $url)
    {
        if (preg_match('/\/(\d+)$/', $url, $matches)) {
            return (int)$matches[1];
        }
        return false;
    }

    /**
     * 提取符合正则表达式的值（只保留数字和小数点）
     */
    protected function extractPatternValue(string $text, string $pattern): string
    {
        $value = '';
        if ($text && preg_match($pattern, $text, $m)) {
            $value = trim(preg_replace('/[^0-9.]/', '', $m[1]));
        }
        return $value;
    }

    /**
     * 从参数列表中获取指定名称的值
     */
    public function getParamValue(array $params, string $name): ?string
    {
        foreach ($params as $item) {
            if (($item['name'] ?? null) === $name) {
                return $item['value'] ?? null;
            }
        }
        return null;
    }

    /**
     * 从 che168 页面上收集图片 URL
     *
     * @param \QL\QueryList $ql
     * @param string        $selector           选择器
     * @param bool          $preferDataOriginal 为 true 时优先用 data-original（#pic_li 懒加载）
     * @return array<string>
     */
    protected function collectChe168Images($ql, string $selector, bool $preferDataOriginal): array
    {
        return $ql->find($selector)->map(function ($img) use ($preferDataOriginal) {
            $src = $preferDataOriginal ? ($img->attr('data-original') ?: $img->attr('src')) : ($img->attr('src') ?: $img->attr('data-original'));
            if (!$src || $src === '' || str_contains($src, 'default-che168')) {
                return null;
            }
            if (str_starts_with($src, '//')) {
                $src = 'https:' . $src;
            }
            return $src;
        })->filter()->values()->all();
    }

    /**
     * 将 che168 页面 (gb2312/gbk) 转为 UTF-8，避免 Linux 下 DOM 解析异常
     *
     * @param string $html
     * @return string
     */
    protected function ensureUtf8ForChe168(string $html): string
    {
        if (!function_exists('mb_convert_encoding')) {
            return $html;
        }
        $isGb = stripos($html, 'charset=gb2312') !== false || stripos($html, 'charset=gbk') !== false;
        if (!$isGb) {
            return $html;
        }
        $out = @mb_convert_encoding($html, 'UTF-8', 'GB2312');
        if ($out === false || $out === '') {
            $out = @mb_convert_encoding($html, 'UTF-8', 'GBK');
        }
        if ($out !== false && $out !== '') {
            $html = $out;
            $html = preg_replace('/charset\s*=\s*["\']?gb(2312|k)["\']?/i', 'charset="utf-8"', $html);
        }
        return $html;
    }

    /**
     * 安全解 JSON
     *
     * @param string $json
     * @param string $context
     * @return array
     * @throws BusinessException
     */
    protected function decodeJson(string $json, string $context = 'JSON'): array
    {
        $data = json_decode($json, true);
        if (json_last_error() !== JSON_ERROR_NONE || !is_array($data)) {
            throw new BusinessException($context . ' 解析失败：' . json_last_error_msg());
        }
        return $data;
    }
}