857 lines
27 KiB
PHP
857 lines
27 KiB
PHP
<?php
|
||
|
||
namespace app\common\utils\crawl\service;
|
||
|
||
use app\common\model\UsedcarCrawlResultsImages;
|
||
use app\common\service\UploadService;
|
||
use GuzzleHttp\Client;
|
||
use GuzzleHttp\Exception\GuzzleException;
|
||
use QL\QueryList;
|
||
use support\Redis;
|
||
use support\Db;
|
||
use support\exception\BusinessException;
|
||
use Throwable;
|
||
|
||
class UsedCarCrawl
|
||
{
|
||
// 默认请求头
|
||
private const DEFAULT_HEADERS = [
|
||
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||
'Accept-Language' => 'zh-CN,zh;q=0.9',
|
||
'Referer' => 'https://www.dongchedi.com/',
|
||
];
|
||
|
||
// Redis队列键名
|
||
private const REDIS_QUEUE_KEY = 'usedcar:urls';
|
||
|
||
// 能源类型常量
|
||
public const ENERGY_TYPE_RENEWABLE = 'RE';
|
||
public const ENERGY_TYPE_CONVENTIONAL = 'CE';
|
||
public const ENERGY_TYPE_UNKNOWN = 'UNKNOWN';
|
||
|
||
/**
|
||
* 插入队列url
|
||
*
|
||
* @param string $url URL地址
|
||
* @param string $scene 场景：usedcar(二手信息) config(配置信息)
|
||
* @param int $id 记录ID
|
||
* @return int
|
||
*/
|
||
public function pushUrl(string $url = '', string $scene = 'usedcar', int $id = 0): int
|
||
{
|
||
$data = [
|
||
'scene' => $scene,
|
||
'url' => $url,
|
||
'id' => $id,
|
||
];
|
||
|
||
return Redis::lpush(
|
||
self::REDIS_QUEUE_KEY,
|
||
json_encode($data, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES)
|
||
);
|
||
}
|
||
|
||
/**
|
||
* 获取队列url
|
||
*
|
||
* @return array|null
|
||
* @throws BusinessException
|
||
*/
|
||
public function popUrl(): ?array
|
||
{
|
||
$data = Redis::rPop(self::REDIS_QUEUE_KEY);
|
||
if (!$data) {
|
||
return null;
|
||
}
|
||
|
||
$task = json_decode($data, true);
|
||
if (json_last_error() !== JSON_ERROR_NONE || !is_array($task)) {
|
||
throw new BusinessException('任务数据解析失败：' . json_last_error_msg());
|
||
}
|
||
|
||
// 基本字段校验
|
||
if (empty($task['url']) || empty($task['scene'])) {
|
||
throw new BusinessException('队列任务数据缺少必要字段：url 或 scene');
|
||
}
|
||
|
||
return $task;
|
||
}
|
||
|
||
/**
|
||
* 执行抓取任务
|
||
*
|
||
* @return bool
|
||
* @throws BusinessException
|
||
*/
|
||
public function execute(): bool
|
||
{
|
||
try {
|
||
$task = $this->popUrl();
|
||
|
||
if (!$task) {
|
||
// 队列为空，任务结束
|
||
return false;
|
||
}
|
||
|
||
$url = (string)$task['url'];
|
||
$scene = (string)$task['scene'];
|
||
$id = (int)($task['id'] ?? 0);
|
||
|
||
$html = $this->fetchHtml($url);
|
||
if ($html === '') {
|
||
throw new BusinessException("页面内容为空: {$url}");
|
||
}
|
||
|
||
switch ($scene) {
|
||
case 'usedcar':
|
||
$info = $this->parseUsedCarDetail($html, $url);
|
||
$insertId = $this->saveUsedCarBasicInfo($info);
|
||
|
||
// 下载图片 + 上传 + 入库（图片失败不影响主记录）
|
||
$imagesArray = json_decode($info['images'] ?? '[]', true);
|
||
if (json_last_error() !== JSON_ERROR_NONE) {
|
||
$imagesArray = [];
|
||
}
|
||
$saveDir = public_path('uploads/usedcar/' . $insertId);
|
||
$this->uploadAndRemove($imagesArray, $saveDir, $info['used_car_id'], $insertId);
|
||
|
||
// 继续派发配置页任务
|
||
if (!empty($info['car_id_url'])) {
|
||
$this->pushUrl($info['car_id_url'], 'config', $insertId);
|
||
}
|
||
break;
|
||
|
||
case 'config':
|
||
if ($id <= 0) {
|
||
throw new BusinessException("配置任务缺少主记录ID");
|
||
}
|
||
|
||
$data = $this->parseConfigDetail($html);
|
||
$this->saveUsedCarConfigInfo($data, $id);
|
||
break;
|
||
|
||
default:
|
||
throw new BusinessException("未知的任务场景: " . $scene);
|
||
}
|
||
|
||
return true;
|
||
} catch (GuzzleException $e) {
|
||
throw new BusinessException("网络请求失败: " . $e->getMessage());
|
||
} catch (BusinessException $e) {
|
||
// 业务异常原样抛出，方便上层知道是逻辑问题
|
||
throw $e;
|
||
} catch (Throwable $th) {
|
||
// 兜底异常，避免 worker 直接挂掉
|
||
throw new BusinessException(
|
||
"执行任务出错: " . $th->getMessage()
|
||
. ' | 行号: ' . $th->getLine()
|
||
. ' | 文件: ' . $th->getFile()
|
||
);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 获取网页HTML内容
|
||
*
|
||
* @param string $url
|
||
* @return string
|
||
* @throws GuzzleException
|
||
*/
|
||
public function fetchHtml(string $url): string
|
||
{
|
||
$client = new Client();
|
||
$response = $client->request('GET', $url, [
|
||
'headers' => self::DEFAULT_HEADERS,
|
||
'timeout' => 10,
|
||
]);
|
||
|
||
return (string)$response->getBody();
|
||
}
|
||
|
||
/**
|
||
* 解析二手车详情页信息
|
||
*
|
||
* @param string $html
|
||
* @param string $url
|
||
* @return array
|
||
* @throws BusinessException
|
||
*/
|
||
public function parseUsedCarDetail(string $html = '', string $url = ''): array
|
||
{
|
||
try {
|
||
$ql = QueryList::html($html);
|
||
} catch (Throwable $e) {
|
||
throw new BusinessException('解析 HTML 失败（详情页）: ' . $e->getMessage());
|
||
}
|
||
|
||
$title = trim($ql->find('h1')->text());
|
||
$price = trim($ql->find('.head-info_price-wrap__Y4bxi > p')->text());
|
||
$pageText = $ql->find('body')->text();
|
||
$jsonText = $ql->find('#__NEXT_DATA__')->text();
|
||
|
||
if ($jsonText === '') {
|
||
throw new BusinessException('详情页未找到 __NEXT_DATA__ JSON');
|
||
}
|
||
|
||
$jsonArray = $this->decodeJson($jsonText, '详情页 __NEXT_DATA__');
|
||
|
||
// 车辆相册
|
||
$images = $jsonArray['props']['pageProps']['skuDetail']['head_images'] ?? [];
|
||
|
||
// 车辆ID / 年份
|
||
$carInfo = $jsonArray['props']['pageProps']['skuDetail']['car_info'] ?? [];
|
||
$carId = $carInfo['car_id'] ?? 0;
|
||
$year = (int)($carInfo['year'] ?? 0);
|
||
|
||
// 车辆ID URL
|
||
$carIdUrl = $carId ? ('https://www.dongchedi.com/auto/params-carIds-' . $carId) : '';
|
||
|
||
// 能源类型（两个字段择一）
|
||
$overview = $jsonArray['props']['pageProps']['skuDetail']['car_config_overview'] ?? [];
|
||
$fuelForm = $overview['power']['fuel_form'] ?? '';
|
||
$newFuelForm = $overview['new_energy_power']['fuel_form'] ?? '';
|
||
$resultFuelForm = $fuelForm === '' ? $newFuelForm : $fuelForm;
|
||
|
||
// 使用里程
|
||
$importantText = $jsonArray['props']['pageProps']['skuDetail']['important_text'] ?? '';
|
||
$importantTextArray = $importantText ? explode(' | ', $importantText) : [];
|
||
$usageMileage = '';
|
||
if (isset($importantTextArray[1])) {
|
||
$usageMileage = str_replace('万公里', '', $importantTextArray[1]);
|
||
}
|
||
|
||
// 车辆名称 品牌、车系、车型
|
||
$brandName = $carInfo['brand_name'] ?? '';
|
||
$seriesName = $carInfo['series_name'] ?? '';
|
||
$carName = $carInfo['car_name'] ?? '';
|
||
|
||
// 商家信息
|
||
$shopInfo = $jsonArray['props']['pageProps']['skuDetail']['shop_info'] ?? [];
|
||
$formatShopInfo = [
|
||
'shop_id' => $shopInfo['shop_id'] ?? 0,
|
||
'shop_name' => $shopInfo['shop_short_name'] ?? '',
|
||
'shop_address' => $shopInfo['shop_address'] ?? '',
|
||
];
|
||
|
||
// 简介参数
|
||
$otherParams = $jsonArray['props']['pageProps']['skuDetail']['other_params'] ?? [];
|
||
$displacement = $this->getParamValue($otherParams, '排量');
|
||
$gearbox = $this->getParamValue($otherParams, '变速箱');
|
||
|
||
$formatCarInfo = [
|
||
'displacement' => $displacement,
|
||
'gearbox' => $gearbox,
|
||
];
|
||
|
||
// 提取新车指导价 / 比新车省金额
|
||
$newPrice = $this->extractPatternValue($pageText, '/新车指导价[:：]\s*([^\r\n万]*万)/u');
|
||
$priceDifference = $this->extractPatternValue($pageText, '/比新车省[:：]\s*([^\r\n万]*万)/u');
|
||
|
||
// 提取车况介绍
|
||
$conditionDesc = '';
|
||
if ($pageText && preg_match('/车况介绍\s*(.*?)\s*推荐理由/su', $pageText, $m)) {
|
||
$conditionDesc = trim($m[1]);
|
||
}
|
||
|
||
return [
|
||
'used_car_id' => $this->extractIdFromUrl($url),
|
||
'car_id' => $carId,
|
||
'url' => $url,
|
||
'title' => $title,
|
||
'price' => str_replace('万', '', $this->decodePrice($price)),
|
||
'new_price' => str_replace('万', '', $newPrice),
|
||
'price_difference' => str_replace('万', '', $priceDifference),
|
||
'condition_desc' => $conditionDesc,
|
||
'images' => json_encode($images, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
||
'car_id_url' => $carIdUrl,
|
||
'used_detail' => $jsonText,
|
||
'format_shop_info' => json_encode($formatShopInfo, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
||
'fuel_form' => $resultFuelForm,
|
||
'classify_energy' => $this->detectEnergyType($resultFuelForm),
|
||
'usage_mileage' => $usageMileage,
|
||
'brand_name' => $brandName,
|
||
'series_name' => $seriesName,
|
||
'car_name' => $carName,
|
||
'year' => $year,
|
||
'format_car_info' => json_encode($formatCarInfo, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
||
];
|
||
}
|
||
|
||
/**
|
||
* 解析配置详情页信息
|
||
*
|
||
* @param string $html
|
||
* @return array
|
||
* @throws BusinessException
|
||
*/
|
||
public function parseConfigDetail(string $html = ''): array
|
||
{
|
||
try {
|
||
$ql = QueryList::html($html);
|
||
} catch (Throwable $e) {
|
||
throw new BusinessException('解析 HTML 失败（配置页）: ' . $e->getMessage());
|
||
}
|
||
|
||
$jsonText = $ql->find('#__NEXT_DATA__')->text();
|
||
if ($jsonText === '') {
|
||
throw new BusinessException('配置页未找到 __NEXT_DATA__ JSON');
|
||
}
|
||
|
||
$raw = $this->decodeJson($jsonText, '配置页 __NEXT_DATA__');
|
||
|
||
$properties = $raw['props']['pageProps']['rawData']['properties'] ?? [];
|
||
$carInfo = $raw['props']['pageProps']['rawData']['car_info'][0] ?? [];
|
||
|
||
$valuesMap = $this->buildValuesMap($carInfo);
|
||
$groups = $this->buildCarSpecGroups($properties, $valuesMap);
|
||
|
||
return [
|
||
'format_car_info' => [
|
||
'market_time' => $this->getFieldValue($groups, 'market_time'), // 上市时间
|
||
'driver_form' => $this->getFieldValue($groups, 'driver_form'), // 驱动形式
|
||
'seat_count' => $this->getFieldValue($groups, 'seat_count'), // 座位数
|
||
'engine_description' => $this->getFieldValue($groups, 'engine_description'), // 发动机
|
||
'engine_max_power_horsepower' => $this->getFieldValue($groups, 'engine_max_power_horsepower'), // 发动机最大功率
|
||
'electric_description' => $this->getFieldValue($groups, 'electric_description'), // 电动机
|
||
'battery_capacity' => $this->getFieldValue($groups, 'battery_capacity'), // 电池容量
|
||
'battery_type' => $this->getFieldValue($groups, 'battery_type'), // 电池类型
|
||
'recharge_mileage' => $this->getFieldValue($groups, 'recharge_mileage'), // 续航里程
|
||
],
|
||
'groups' => $groups,
|
||
];
|
||
}
|
||
|
||
/**
|
||
* 从车辆配置中获取能源类型
|
||
*
|
||
* @param array $data JSON数据
|
||
* @return string|null 能源类型
|
||
*/
|
||
public function getFuelFormByConfig(array $data): ?string
|
||
{
|
||
if (empty($data[0]['items']) || !is_array($data[0]['items'])) {
|
||
return null;
|
||
}
|
||
|
||
foreach ($data[0]['items'] as $item) {
|
||
if (isset($item['key']) && $item['key'] === 'fuel_form') {
|
||
return $item['value'] ?? null;
|
||
}
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
/**
|
||
* 从数据中根据字段key获取值
|
||
*
|
||
* @param mixed $data 数据
|
||
* @param string $field 字段名
|
||
* @return string|null
|
||
*/
|
||
public function getFieldValue($data, string $field): ?string
|
||
{
|
||
if (is_string($data)) {
|
||
$data = json_decode($data, true);
|
||
if (json_last_error() !== JSON_ERROR_NONE) {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
if (!is_array($data)) {
|
||
return null;
|
||
}
|
||
|
||
foreach ($data as $group) {
|
||
if (isset($group['items']) && is_array($group['items'])) {
|
||
foreach ($group['items'] as $item) {
|
||
if (($item['key'] ?? null) === $field) {
|
||
return $item['value'] ?? null;
|
||
}
|
||
|
||
if (!empty($item['children']) && is_array($item['children'])) {
|
||
foreach ($item['children'] as $child) {
|
||
if (($child['key'] ?? null) === $field) {
|
||
return $child['value'] ?? null;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
/**
|
||
* 从参数列表中获取指定名称的值
|
||
*/
|
||
public function getParamValue(array $params, string $name): ?string
|
||
{
|
||
foreach ($params as $item) {
|
||
if (($item['name'] ?? null) === $name) {
|
||
return $item['value'] ?? null;
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
|
||
/**
|
||
* 判断车辆能源类型
|
||
*/
|
||
public function detectEnergyType(string $text): string
|
||
{
|
||
if (function_exists('mb_internal_encoding')) {
|
||
mb_internal_encoding('UTF-8');
|
||
}
|
||
|
||
// 明确定义的新能源关键字
|
||
$newEnergyKeywords = [
|
||
'纯电', '纯电动', '电动车', 'BEV',
|
||
'插电混动', '插电式混合', '插电式', '插电', 'PHEV',
|
||
'增程式', '增程', 'REEV',
|
||
'氢能', '氢燃料', '燃料电池', 'FCEV',
|
||
'太阳能',
|
||
];
|
||
|
||
// 传统能源关键字
|
||
$traditionalKeywords = [
|
||
'汽油',
|
||
'柴油',
|
||
'天然气', 'LNG', 'CNG',
|
||
];
|
||
|
||
// 普通混动关键字（归为传统能源）
|
||
$hybridKeywords = [
|
||
'油电混合', '油电混动', '油电',
|
||
'混动',
|
||
'双擎',
|
||
'HEV',
|
||
];
|
||
|
||
$contains = function (string $haystack, string $needle): bool {
|
||
if ($needle === '') {
|
||
return false;
|
||
}
|
||
if (function_exists('mb_stripos')) {
|
||
return mb_stripos($haystack, $needle) !== false;
|
||
}
|
||
return stripos($haystack, $needle) !== false;
|
||
};
|
||
|
||
foreach ($newEnergyKeywords as $kw) {
|
||
if ($contains($text, $kw)) {
|
||
return self::ENERGY_TYPE_RENEWABLE;
|
||
}
|
||
}
|
||
|
||
foreach ($hybridKeywords as $kw) {
|
||
if ($contains($text, $kw)) {
|
||
return self::ENERGY_TYPE_CONVENTIONAL;
|
||
}
|
||
}
|
||
|
||
foreach ($traditionalKeywords as $kw) {
|
||
if ($contains($text, $kw)) {
|
||
return self::ENERGY_TYPE_CONVENTIONAL;
|
||
}
|
||
}
|
||
|
||
return self::ENERGY_TYPE_UNKNOWN;
|
||
}
|
||
|
||
/**
|
||
* 保存二手车基本信息到数据库
|
||
*
|
||
* @param array $data 数据
|
||
* @return int 插入记录ID
|
||
* @throws BusinessException
|
||
*/
|
||
public function saveUsedCarBasicInfo(array $data): int
|
||
{
|
||
try {
|
||
$time = time();
|
||
$data['created_at'] = $data['created_at'] ?? $time;
|
||
$data['updated_at'] = $time;
|
||
|
||
return (int)Db::table('usedcar_crawl_results')->insertGetId($data);
|
||
} catch (Throwable $e) {
|
||
throw new BusinessException('保存二手车基础信息失败：' . $e->getMessage());
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 更新配置信息到数据库
|
||
*
|
||
* @param array $data 配置数据
|
||
* @param int $id 记录ID
|
||
* @return bool|int
|
||
* @throws BusinessException
|
||
*/
|
||
public function saveUsedCarConfigInfo(array $data, int $id)
|
||
{
|
||
try {
|
||
$record = Db::table('usedcar_crawl_results')->find($id);
|
||
|
||
if (!$record) {
|
||
// 找不到记录就算了，不抛致命错误，视业务情况也可以抛异常
|
||
return false;
|
||
}
|
||
|
||
$groups = $data['groups'] ?? [];
|
||
$oldFormat = [];
|
||
|
||
if (!empty($record->format_car_info)) {
|
||
$decoded = json_decode($record->format_car_info, true);
|
||
if (json_last_error() === JSON_ERROR_NONE && is_array($decoded)) {
|
||
$oldFormat = $decoded;
|
||
}
|
||
}
|
||
|
||
$formatCarInfo = array_merge(
|
||
$oldFormat,
|
||
$data['format_car_info'] ?? []
|
||
);
|
||
|
||
return Db::table('usedcar_crawl_results')
|
||
->where('id', $id)
|
||
->update([
|
||
'format_car_info' => json_encode($formatCarInfo, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
||
'config_detail' => json_encode($groups, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
||
'updated_at' => time(),
|
||
]);
|
||
} catch (Throwable $e) {
|
||
throw new BusinessException('保存车辆配置信息失败：' . $e->getMessage());
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 构建值映射表
|
||
*/
|
||
public function buildValuesMap(array $carInfo): array
|
||
{
|
||
$valuesMap = [];
|
||
|
||
// 顶层字段
|
||
foreach ($carInfo as $k => $v) {
|
||
if ($k === 'info' || is_array($v)) {
|
||
continue;
|
||
}
|
||
$valuesMap[$k] = ['value' => $v];
|
||
}
|
||
|
||
// info中的配置项
|
||
if (!empty($carInfo['info']) && is_array($carInfo['info'])) {
|
||
foreach ($carInfo['info'] as $k => $v) {
|
||
if (is_array($v) && array_key_exists('value', $v)) {
|
||
$valuesMap[$k] = $v;
|
||
}
|
||
}
|
||
}
|
||
|
||
return $valuesMap;
|
||
}
|
||
|
||
/**
|
||
* 构建车辆规格分组
|
||
*/
|
||
public function buildCarSpecGroups(array $properties, array $valuesMap): array
|
||
{
|
||
$groups = [];
|
||
$currentKey = null;
|
||
|
||
foreach ($properties as $item) {
|
||
$type = (int)($item['type'] ?? 1);
|
||
$key = $item['key'] ?? '';
|
||
$text = $item['text'] ?? '';
|
||
$subList = $item['sub_list'] ?? null;
|
||
|
||
// 分类字段：type = 0 / 2
|
||
if (in_array($type, [0, 2], true)) {
|
||
$currentKey = $key ?: ($text ?: ('group_' . count($groups)));
|
||
|
||
if (!isset($groups[$currentKey])) {
|
||
$groups[$currentKey] = [
|
||
'group_key' => $currentKey,
|
||
'group_text' => $text,
|
||
'group_type' => $type,
|
||
'items' => [],
|
||
];
|
||
}
|
||
|
||
continue;
|
||
}
|
||
|
||
// 普通字段：挂到最近的分类下
|
||
if ($currentKey === null) {
|
||
$currentKey = '_default';
|
||
$groups[$currentKey] = [
|
||
'group_key' => '_default',
|
||
'group_text' => '其他',
|
||
'group_type' => -1,
|
||
'items' => [],
|
||
];
|
||
}
|
||
|
||
$value = '';
|
||
if ($key && isset($valuesMap[$key])) {
|
||
$value = trim((string)($valuesMap[$key]['value'] ?? ''));
|
||
}
|
||
|
||
$node = [
|
||
'key' => $key,
|
||
'text' => $text,
|
||
'value' => $value,
|
||
];
|
||
|
||
if (is_array($subList) && !empty($subList)) {
|
||
$children = [];
|
||
foreach ($subList as $sub) {
|
||
$subKey = $sub['key'] ?? '';
|
||
$subText = $sub['text'] ?? '';
|
||
$subValue = '';
|
||
|
||
if ($subKey && isset($valuesMap[$subKey])) {
|
||
$subValue = trim((string)($valuesMap[$subKey]['value'] ?? ''));
|
||
}
|
||
|
||
$children[] = [
|
||
'key' => $subKey,
|
||
'text' => $subText,
|
||
'value' => $subValue,
|
||
];
|
||
}
|
||
|
||
$node['children'] = $children;
|
||
}
|
||
|
||
$groups[$currentKey]['items'][] = $node;
|
||
}
|
||
|
||
return array_values($groups);
|
||
}
|
||
|
||
/**
|
||
* 解码懂车帝价格
|
||
*/
|
||
protected function decodePrice(string $str): string
|
||
{
|
||
$map = [
|
||
"\u{E53D}" => '1',
|
||
"\u{E3F0}" => '2',
|
||
"\u{E422}" => '3',
|
||
"\u{E42C}" => '4',
|
||
"\u{E49C}" => '5',
|
||
"\u{E42B}" => '6',
|
||
"\u{E4FE}" => '7',
|
||
"\u{E548}" => '8',
|
||
"\u{E4C8}" => '9',
|
||
"\u{E453}" => '0',
|
||
"\u{E45F}" => '万',
|
||
];
|
||
|
||
return strtr($str, $map);
|
||
}
|
||
|
||
/**
|
||
* 从URL中提取ID
|
||
*/
|
||
protected function extractIdFromUrl(string $url)
|
||
{
|
||
if (preg_match('/\/(\d+)$/', $url, $matches)) {
|
||
return (int)$matches[1];
|
||
}
|
||
return false;
|
||
}
|
||
|
||
/**
|
||
* 提取符合正则表达式的值（只保留数字和小数点）
|
||
*/
|
||
protected function extractPatternValue(string $text, string $pattern): string
|
||
{
|
||
$value = '';
|
||
if ($text && preg_match($pattern, $text, $m)) {
|
||
$value = trim(preg_replace('/[^0-9.]/', '', $m[1]));
|
||
}
|
||
return $value;
|
||
}
|
||
|
||
/**
|
||
* 下载单张图片
|
||
*/
|
||
public function downloadImage(string $url, string $saveDir, ?string $filename = null, array $headers = []): ?string
|
||
{
|
||
if (!is_dir($saveDir)) {
|
||
if (!@mkdir($saveDir, 0777, true) && !is_dir($saveDir)) {
|
||
// echo "[Image] 创建目录失败：{$saveDir}\n";
|
||
return null;
|
||
}
|
||
}
|
||
|
||
if ($filename === null) {
|
||
$path = parse_url($url, PHP_URL_PATH);
|
||
$ext = pathinfo($path ?? '', PATHINFO_EXTENSION);
|
||
|
||
if ($ext === '') {
|
||
$ext = 'jpg';
|
||
}
|
||
|
||
$filename = md5($url) . '.' . $ext;
|
||
}
|
||
|
||
$savePath = rtrim($saveDir, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . $filename;
|
||
|
||
$defaultHeaders = [
|
||
'User-Agent: ' . self::DEFAULT_HEADERS['User-Agent'],
|
||
'Accept: image/avif,image/webp,image/apng,image/*,*/*;q=0.8',
|
||
'Referer: ' . self::DEFAULT_HEADERS['Referer'],
|
||
];
|
||
|
||
foreach ($headers as $h) {
|
||
$defaultHeaders[] = $h;
|
||
}
|
||
|
||
$ch = curl_init($url);
|
||
curl_setopt_array($ch, [
|
||
CURLOPT_RETURNTRANSFER => true,
|
||
CURLOPT_FOLLOWLOCATION => true,
|
||
CURLOPT_CONNECTTIMEOUT => 5,
|
||
CURLOPT_TIMEOUT => 15,
|
||
CURLOPT_SSL_VERIFYPEER => false,
|
||
CURLOPT_SSL_VERIFYHOST => false,
|
||
CURLOPT_HTTPHEADER => $defaultHeaders,
|
||
]);
|
||
|
||
$data = curl_exec($ch);
|
||
$errNo = curl_errno($ch);
|
||
$code = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||
curl_close($ch);
|
||
|
||
if ($errNo !== 0 || $code !== 200 || $data === false || $data === '') {
|
||
return null;
|
||
}
|
||
|
||
$bytes = @file_put_contents($savePath, $data);
|
||
if ($bytes === false) {
|
||
return null;
|
||
}
|
||
|
||
return $savePath;
|
||
}
|
||
|
||
/**
|
||
* 批量下载相册
|
||
*
|
||
* @param array $urls 图片URL数组
|
||
* @param string $saveDir 保存目录
|
||
* @return array [原始URL => 保存路径或null]
|
||
*/
|
||
public function downloadImageAlbum(array $urls, string $saveDir): array
|
||
{
|
||
$result = [];
|
||
|
||
foreach ($urls as $url) {
|
||
$url = trim((string)$url);
|
||
if ($url === '') {
|
||
continue;
|
||
}
|
||
|
||
if (strpos($url, '//') === 0) {
|
||
$url = 'https:' . $url;
|
||
}
|
||
|
||
try {
|
||
$path = $this->downloadImage($url, $saveDir);
|
||
} catch (Throwable $e) {
|
||
$path = null;
|
||
}
|
||
|
||
$result[$url] = $path;
|
||
}
|
||
|
||
return $result;
|
||
}
|
||
|
||
/**
|
||
* 下载 -> 上传 -> 删除本地
|
||
*
|
||
* @param array $images 原始相册URL列表
|
||
* @param string $path 本地保存目录
|
||
* @param int $usedCarId 懂车帝 used_car_id
|
||
* @param int $id 本地 crawl 记录id
|
||
* @return bool
|
||
*/
|
||
public function uploadAndRemove(array $images, string $path, $usedCarId, $id): bool
|
||
{
|
||
if (empty($images)) {
|
||
return false;
|
||
}
|
||
|
||
// 下载图片
|
||
$filePatch = $this->downloadImageAlbum($images, $path);
|
||
if (empty($filePatch)) {
|
||
return false;
|
||
}
|
||
|
||
$uploadService = new UploadService();
|
||
|
||
foreach ($filePatch as $url => $filePath) {
|
||
if ($filePath === null) {
|
||
continue;
|
||
}
|
||
|
||
try {
|
||
// 上传云端
|
||
$param = [
|
||
'path' => $filePath,
|
||
'scene' => 'usedcar_crawl',
|
||
'category' => 'usedcar',
|
||
];
|
||
|
||
$result = $uploadService->uploadLocalFile($param);
|
||
if (empty($result['file_name'])) {
|
||
// 上传失败则跳过，但不抛异常影响主流程
|
||
continue;
|
||
}
|
||
|
||
// 保存图片信息
|
||
UsedcarCrawlResultsImages::create([
|
||
'uc_id' => $id,
|
||
'used_car_id' => $usedCarId,
|
||
'original_name'=> $result['original_name'] ?? '',
|
||
'file_path' => $result['file_name'],
|
||
]);
|
||
} catch (Throwable $e) {
|
||
// 单张图失败不影响其他
|
||
// 可以按需写日志
|
||
} finally {
|
||
// 删除本地文件
|
||
if (is_file($filePath)) {
|
||
@unlink($filePath);
|
||
}
|
||
}
|
||
}
|
||
|
||
// 删除目录
|
||
rmdir($path);
|
||
|
||
return true;
|
||
}
|
||
|
||
/**
|
||
* 安全解 JSON
|
||
*
|
||
* @param string $json
|
||
* @param string $context
|
||
* @return array
|
||
* @throws BusinessException
|
||
*/
|
||
protected function decodeJson(string $json, string $context = 'JSON'): array
|
||
{
|
||
$data = json_decode($json, true);
|
||
if (json_last_error() !== JSON_ERROR_NONE || !is_array($data)) {
|
||
throw new BusinessException($context . ' 解析失败：' . json_last_error_msg());
|
||
}
|
||
return $data;
|
||
}
|
||
} |