x-server-mobile/backend/app/service/UsedCarCrawlService.php

<?php

namespace app\Service;

use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException;
use QL\QueryList;
use support\Redis;

/**
 * 二手车数据抓取服务
 * 支持从懂车帝、汽车之家等网站抓取车辆信息
 */
class UsedCarCrawlService
{
    private Client $httpClient;

    // 默认请求头
    private const DEFAULT_HEADERS = [
        'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
        'Accept-Language' => 'zh-CN,zh;q=0.9',
        'Referer' => 'https://www.dongchedi.com/',
    ];

    public function __construct()
    {
        $this->httpClient = new Client([
            'verify' => false,
            'timeout' => 15,
        ]);
    }

    /**
     * 选择抓取方式并执行
     *
     * @param string $url 车辆详情页URL
     * @return array 抓取结果
     * @throws \Exception
     */
    public function crawlByUrl(string $url): array
    {
        // 根据URL判断来源并选择抓取方法
        if (str_contains($url, 'dongchedi.com')) {
            // 懂车帝PC端
            return $this->crawlDongchedi($url);
        } elseif (str_contains($url, 'dcdapp.com')) {
            // 懂车帝移动端 - 需要先转换为PC端
            $pcUrl = $this->getDongchediPcUrl($url);
            if (!$pcUrl) {
                throw new \Exception('无法获取懂车帝PC端链接');
            }
            return $this->crawlDongchedi($pcUrl);
        } elseif (str_contains($url, 't.zijieimg.com')) {
            // 懂车帝分享短链
            $finalUrl = $this->getFinalUrl($url);
            $pcUrl = $this->getDongchediPcUrl($finalUrl);
            if (!$pcUrl) {
                throw new \Exception('无法获取懂车帝PC端链接');
            }
            return $this->crawlDongchedi($pcUrl);
        } elseif (str_contains($url, 'che168.com')) {
            // 汽车之家
            return $this->crawlChe168($url);
        }

        throw new \Exception('暂不支持此网站的数据抓取');
    }

    /**
     * 获取短链的最终URL
     */
    private function getFinalUrl(string $url): string
    {
        try {
            $response = $this->httpClient->get($url, [
                'allow_redirects' => ['track_redirects' => true],
            ]);

            $history = $response->getHeader('X-Guzzle-Redirect-History');
            return end($history) ?: $url;
        } catch (\Exception $e) {
            throw new \Exception('获取短链失败: ' . $e->getMessage());
        }
    }

    /**
     * 从懂车帝移动端URL获取PC端URL
     */
    private function getDongchediPcUrl(string $url): ?string
    {
        $query = parse_url($url, PHP_URL_QUERY);
        if (!$query) {
            return null;
        }

        parse_str($query, $params);
        $skuId = $params['sku_id'] ?? null;

        return $skuId ? 'https://www.dongchedi.com/usedcar/' . $skuId : null;
    }

    /**
     * 抓取懂车帝二手车信息
     */
    private function crawlDongchedi(string $url): array
    {
        try {
            $html = $this->fetchHtml($url);

            if (str_contains($html, '404 Not Found')) {
                throw new \Exception('车辆已下架或找不到页面');
            }

            return $this->parseDongchediHtml($html, $url);
        } catch (\Exception $e) {
            throw new \Exception('抓取懂车帝数据失败: ' . $e->getMessage());
        }
    }

    /**
     * 抓取汽车之家二手车信息
     */
    private function crawlChe168(string $url): array
    {
        try {
            // 处理不同格式的汽车之家URL
            if (str_contains($url, 'home?infoid=')) {
                // PC端链接转换
                $queryString = parse_url($url, PHP_URL_QUERY);
                parse_str($queryString, $params);
                $carId = $params['infoid'] ?? null;
                $url = 'https://www.che168.com/dealer/111/' . $carId . '.html?pvareaid=0';
            } elseif (str_contains($url, 'wap.che168.com/dealer')) {
                // 移动端链接转换
                $url = $this->getChe168PcUrl($url);
            }

            $html = $this->fetchHtml($url);
            return $this->parseChe168Html($html, $url);
        } catch (\Exception $e) {
            throw new \Exception('抓取汽车之家数据失败: ' . $e->getMessage());
        }
    }

    /**
     * 从汽车之家移动端URL获取PC端URL
     */
    private function getChe168PcUrl(string $url): string
    {
        $path = parse_url($url, PHP_URL_PATH);
        $parts = explode('/', $path);
        $filename = end($parts);
        $carId = pathinfo($filename, PATHINFO_FILENAME);

        return 'https://www.che168.com/dealer/111/' . $carId . '.html?pvareaid=0';
    }

    /**
     * 获取网页HTML内容
     */
    private function fetchHtml(string $url): string
    {
        try {
            $response = $this->httpClient->get($url, [
                'headers' => self::DEFAULT_HEADERS,
            ]);
            return (string)$response->getBody();
        } catch (GuzzleException $e) {
            throw new \Exception('获取页面失败: ' . $e->getMessage());
        }
    }

    /**
     * 解析懂车帝HTML
     */
    private function parseDongchediHtml(string $html, string $url): array
    {
        try {
            $ql = QueryList::html($html);

            // 提取JSON数据
            $jsonText = $ql->find('#__NEXT_DATA__')->text();
            if (empty($jsonText)) {
                throw new \Exception('无法找到页面数据');
            }

            $jsonData = json_decode($jsonText, true);
            if (!$jsonData) {
                throw new \Exception('解析JSON数据失败');
            }

            $skuDetail = $jsonData['props']['pageProps']['skuDetail'] ?? [];
            $carInfo = $skuDetail['car_info'] ?? [];

            // 提取价格（懂车帝使用特殊字符编码）
            $priceText = $ql->find('.head-info_price-wrap__Y4bxi > p')->text();
            $price = $this->decodeDongchediPrice($priceText);

            // 提取车辆信息
            $title = $carInfo['car_name'] ?? '';
            $brandName = $carInfo['brand_name'] ?? '';
            $seriesName = $carInfo['series_name'] ?? '';
            $year = $carInfo['year'] ?? '';
            $bodyColor = $carInfo['body_color'] ?? '';

            // 提取配置信息
            $overview = $skuDetail['car_config_overview'] ?? [];
            $power = $overview['power']['horsepower'] ?? '';
            $displacement = $overview['power']['capacity'] ?? '';

            // 提取上牌时间和车源地
            $otherParams = $skuDetail['other_params'] ?? [];
            $plateTime = $this->getParamValue($otherParams, '上牌时间');
            $sourcePlace = $this->getParamValue($otherParams, '车源地');

            // 提取里程
            $importantText = $skuDetail['important_text'] ?? '';
            $mileage = 0;
            if (preg_match('/(\d+\.?\d*)\s*万公里/', $importantText, $matches)) {
                $mileage = (int)($matches[1] * 10000);
            }

            // 提取图片
            $images = $skuDetail['head_images'] ?? [];

            // 提取车况介绍
            $conditionDesc = '';
            if (preg_match('/车况介绍\s*(.*?)\s*推荐理由/s', $html, $matches)) {
                $conditionDesc = trim($matches[1]);
            }

            // 提取车辆ID
            $carId = $carInfo['car_id'] ?? 0;
            $brandId = $carInfo['brand_id'] ?? 0;
            $seriesId = $carInfo['series_id'] ?? 0;
            $usedCarId = $this->extractIdFromUrl($url);

            return [
                'title' => $title,
                'price' => $price,
                'brand_name' => $brandName,
                'brand_id' => $brandId,
                'series_name' => $seriesName,
                'series_id' => $seriesId,
                'year' => $year,
                'car_id' => $carId,
                'used_car_id' => $usedCarId,
                'power' => $power,
                'displacement' => $displacement,
                'plate_time' => $plateTime,
                'source_place' => $sourcePlace,
                'body_color' => $bodyColor,
                'mileage' => $mileage,
                'images' => $images,
                'condition_desc' => $conditionDesc,
                'source_url' => $url,
            ];
        } catch (\Exception $e) {
            throw new \Exception('解析懂车帝页面失败: ' . $e->getMessage());
        }
    }

    /**
     * 解析汽车之家HTML
     */
    private function parseChe168Html(string $html, string $url): array
    {
        try {
            // 汽车之家使用 GB2312 编码，需要转换
            $html = $this->convertToUtf8($html);

            $ql = QueryList::html($html);

            // 提取车辆名称
            $productName = $ql->find('div.car-box .car-brand-name')->text();

            // 提取价格
            $price = $ql->find('div.goodstartmoney')->text();
            if (empty($price)) {
                $price = $ql->find('span#overlayPrice')->text();
            }
            $price = str_replace(['¥', '￥', '万'], '', $price);
            $price = is_numeric($price) ? (int)($price * 10000) : 0;

            // 提取基本信息
            $basicInfo = [];
            $ql->find('div#nav1 ul.basic-item-ul li')->each(function($li) use (&$basicInfo) {
                $field = trim($li->find('.item-name')->text());
                $value = trim(str_replace($field, '', $li->text()));

                $fieldMap = [
                    '上牌时间' => 'plate_time',
                    '表显里程' => 'mileage_text',
                    '排       量' => 'displacement',
                    '所  在  地' => 'source_place',
                    '车身颜色' => 'body_color',
                    '车况介绍' => 'condition_desc',
                ];

                if (isset($fieldMap[$field])) {
                    $basicInfo[$fieldMap[$field]] = $value;
                }
            });

            // 处理里程
            $mileage = 0;
            if (isset($basicInfo['mileage_text'])) {
                $mileage = (int)str_replace('万公里', '', $basicInfo['mileage_text']) * 10000;
            }

            // 提取面包屑导航获取品牌和车系
            $breadcrumbs = [];
            $ql->find('.bread-crumbs a')->each(function($a) use (&$breadcrumbs) {
                $breadcrumbs[] = trim($a->text());
            });

            $brandName = $breadcrumbs[3] ?? '';
            $seriesName = $breadcrumbs[4] ?? '';
            $brandName = str_replace('二手', '', $brandName);
            $seriesName = str_replace('二手', '', $seriesName);

            // 提取年份
            $year = '';
            if (preg_match('/(\d{4})款/', $productName, $matches)) {
                $year = $matches[1];
            }

            // 提取图片
            $images = [];
            $ql->find('.swiper-container2 .swiper-slide .jiaodianphotoclick img')->each(function($img) use (&$images) {
                $src = $img->attr('src');
                if ($src && !str_contains($src, 'default-che168')) {
                    $images[] = $src;
                }
            });

            // 提取车辆ID
            $usedCarId = $this->extractIdFromUrl($url);

            return [
                'title' => $productName,
                'product_name' => $productName,
                'price' => $price,
                'brand_name' => $brandName,
                'brand_id' => 0,
                'series_name' => $seriesName,
                'series_id' => 0,
                'year' => $year,
                'car_id' => 0,
                'used_car_id' => $usedCarId,
                'plate_time' => $basicInfo['plate_time'] ?? '',
                'displacement' => $basicInfo['displacement'] ?? '',
                'source_place' => $basicInfo['source_place'] ?? '',
                'body_color' => $basicInfo['body_color'] ?? '',
                'mileage' => $mileage,
                'images' => $images,
                'condition_desc' => $basicInfo['condition_desc'] ?? '',
                'source_url' => $url,
            ];
        } catch (\Exception $e) {
            throw new \Exception('解析汽车之家页面失败: ' . $e->getMessage());
        }
    }

    /**
     * 解码懂车帝价格（特殊字符编码）
     */
    private function decodeDongchediPrice(string $str): float
    {
        $map = [
            "\u{E53D}" => '1',
            "\u{E3F0}" => '2',
            "\u{E422}" => '3',
            "\u{E42C}" => '4',
            "\u{E49C}" => '5',
            "\u{E42B}" => '6',
            "\u{E4FE}" => '7',
            "\u{E548}" => '8',
            "\u{E4C8}" => '9',
            "\u{E453}" => '0',
            "\u{E45F}" => '.',
        ];

        $priceStr = strtr($str, $map);
        $priceStr = str_replace('万', '', $priceStr);

        return (float)trim($priceStr);
    }

    /**
     * 从URL中提取ID
     */
    private function extractIdFromUrl(string $url): int
    {
        if (preg_match('/(\d+)\.html?$/', $url, $matches)) {
            return (int)$matches[1];
        }
        if (preg_match('/\/usedcar\/(\d+)/', $url, $matches)) {
            return (int)$matches[1];
        }
        return 0;
    }

    /**
     * 从参数列表中获取指定名称的值
     */
    private function getParamValue(array $params, string $name): ?string
    {
        foreach ($params as $item) {
            if (($item['name'] ?? null) === $name) {
                return $item['value'] ?? null;
            }
        }
        return null;
    }

    /**
     * 将GB2312编码转换为UTF-8
     */
    private function convertToUtf8(string $html): string
    {
        if (!function_exists('mb_convert_encoding')) {
            return $html;
        }

        $isGb = stripos($html, 'charset=gb2312') !== false || stripos($html, 'charset=gbk') !== false;
        if (!$isGb) {
            return $html;
        }

        $out = @mb_convert_encoding($html, 'UTF-8', 'GB2312');
        if ($out === false || $out === '') {
            $out = @mb_convert_encoding($html, 'UTF-8', 'GBK');
        }

        if ($out !== false && $out !== '') {
            $html = preg_replace('/charset\s*=\s*["\']?gb(2312|k)["\']?/i', 'charset="utf-8"', $html);
            return $out;
        }

        return $html;
    }

    /**
     * 下载图片到本地
     *
     * @param string $url 图片URL
     * @param string $saveDir 保存目录
     * @return string|null 保存的文件路径，失败返回null
     */
    public function downloadImage(string $url, string $saveDir): ?string
    {
        try {
            if (!is_dir($saveDir)) {
                mkdir($saveDir, 0755, true);
            }

            $filename = md5($url) . '.jpg';
            $savePath = rtrim($saveDir, '/') . '/' . $filename;

            // 如果文件已存在，直接返回
            if (file_exists($savePath)) {
                return $savePath;
            }

            // 使用 curl 下载图片
            $ch = curl_init($url);
            curl_setopt_array($ch, [
                CURLOPT_RETURNTRANSFER => true,
                CURLOPT_FOLLOWLOCATION => true,
                CURLOPT_CONNECTTIMEOUT => 5,
                CURLOPT_TIMEOUT => 15,
                CURLOPT_SSL_VERIFYPEER => false,
                CURLOPT_SSL_VERIFYHOST => false,
                CURLOPT_HTTPHEADER => [
                    'User-Agent: ' . self::DEFAULT_HEADERS['User-Agent'],
                    'Referer: ' . self::DEFAULT_HEADERS['Referer'],
                ],
            ]);

            $data = curl_exec($ch);
            $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
            curl_close($ch);

            if ($httpCode !== 200 || $data === false) {
                return null;
            }

            file_put_contents($savePath, $data);
            return $savePath;
        } catch (\Exception $e) {
            error_log('下载图片失败: ' . $e->getMessage());
            return null;
        }
    }

    /**
     * 批量下载图片
     *
     * @param array $urls 图片URL数组
     * @param string $saveDir 保存目录
     * @return array [url => localPath]
     */
    public function downloadImages(array $urls, string $saveDir): array
    {
        $result = [];
        foreach ($urls as $url) {
            $url = trim($url);
            if (empty($url)) {
                continue;
            }

            // 处理协议相对URL
            if (str_starts_with($url, '//')) {
                $url = 'https:' . $url;
            }

            $localPath = $this->downloadImage($url, $saveDir);
            $result[$url] = $localPath;
        }

        return $result;
    }

    /**
     * 通过关键词搜索懂车帝车系信息
     *
     * @param string $keyword 搜索关键词（如 "奥迪A4"）
     * @return array 品牌和车系信息
     * @throws \Exception
     */
    public function searchByKeyword(string $keyword): array
    {
        // 访问懂车帝搜索页面获取第一个结果的链接
        $searchUrl = "https://www.dongchedi.com/search?keyword=" . urlencode($keyword);

        try {
            $html = $this->fetchHtml($searchUrl);
            $ql = QueryList::html($html);

            // 查找"查看全部"按钮的链接
            $href = '';
            $ql->find('a.fold-btn')->each(function ($item) use (&$href) {
                $itemHref = $item->attr('href');
                if ($itemHref) {
                    $href = $itemHref;
                }
            });

            if (empty($href)) {
                throw new \Exception('未找到相关车系，请使用更精确的关键词');
            }

            // 提取车系ID
            $seriesId = $this->extractSeriesId($href);
            if (!$seriesId) {
                throw new \Exception('无法解析车系ID');
            }

            // 获取品牌和车系信息
            return $this->getBrandSeries($seriesId);
        } catch (\Exception $e) {
            throw new \Exception('搜索失败: ' . $e->getMessage());
        }
    }

    /**
     * 获取品牌和车系名称
     *
     * @param string $seriesId 车系ID
     * @return array 包含品牌和车系信息的数组
     * @throws \Exception
     */
    public function getBrandSeries(string $seriesId): array
    {
        $url = "https://www.dongchedi.com/auto/series/" . $seriesId . "?anchorId=carModels";

        try {
            $html = $this->fetchHtml($url);
            $ql = QueryList::html($html);

            // 从面包屑导航中提取品牌和车系
            $breadcrumbs = [];
            $ql->find('div.breadcrumb_wrap__2GLeU span')->each(function ($item) use (&$breadcrumbs) {
                $breadcrumbs[] = trim($item->text());
            });

            $brand = $breadcrumbs[2] ?? '';
            $series = $breadcrumbs[3] ?? '';

            // 获取车型列表
            $models = $this->getSeriesModels($seriesId);

            return [
                'brand' => $brand,
                'series' => $series,
                'series_id' => $seriesId,
                'models' => $models,
            ];
        } catch (\Exception $e) {
            throw new \Exception('获取车系信息失败: ' . $e->getMessage());
        }
    }

    /**
     * 获取车系下的车型列表
     *
     * @param string $seriesId 车系ID
     * @return array 车型列表
     */
    public function getSeriesModels(string $seriesId): array
    {
        $url = "https://www.dongchedi.com/auto/series/" . $seriesId . "?anchorId=carModels";
        $models = [];

        try {
            $html = $this->fetchHtml($url);
            $ql = QueryList::html($html);

            // 使用 Tailwind CSS 类名选择器（懂车帝使用 Tailwind）
            $nodes = $ql->find('div.tw-grid div.tw-relative div.tw-items-center>a');
            $texts = $nodes->texts()->all();
            $links = $nodes->attrs('href')->all();

            // 提取价格信息（可选）
            $priceNodes = $ql->find('div.tw-grid div.tw-text-color-gray-800');
            $prices = $priceNodes->texts()->all();

            foreach ($texts as $index => $text) {
                $modelName = trim($text);
                if (empty($modelName)) {
                    continue;
                }

                $link = $links[$index] ?? '';
                if (!empty($link) && strpos($link, 'http') !== 0) {
                    $link = 'https://www.dongchedi.com' . $link;
                }

                $priceStr = $prices[$index] ?? '';
                $msrp = 0;
                if (!empty($priceStr)) {
                    $msrp = (int)(str_replace("万", "", $priceStr) * 10000);
                }

                $models[] = [
                    'id' => $index,
                    'name' => $modelName,
                    'url' => $link,
                    'msrp' => $msrp,
                ];
            }

            return $models;
        } catch (\Exception $e) {
            // 车型解析失败，记录日志并返回空数组
            error_log('获取车型列表失败: ' . $e->getMessage());
            return [];
        }
    }

    /**
     * 从懂车帝URL中提取车系ID
     *
     * @param string $url 懂车帝URL
     * @return string|null 车系ID
     */
    private function extractSeriesId(string $url): ?string
    {
        // 匹配 /auto/series/x123/ 格式
        if (preg_match('/\/auto\/series\/(\d+)/', $url, $matches)) {
            return $matches[1];
        }

        // 匹配 /auto/series/x123/params/ 格式
        if (preg_match('/\/auto\/series\/(\d+)/', $url, $matches)) {
            return $matches[1];
        }

        return null;
    }
}