xinyouhe/useds
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

抓取车辆

Browse Source
This commit is contained in:
WIN-C27C5MSVTPO\16961
2026-01-16 18:02:30 +08:00
parent dde8e88106
commit 17701b9262
4 changed files with 471 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/common/controller/UsedCarController.php
+36 -2
View File
@@ -1,7 +1,6 @@
<?php
namespace app\common\controller;
use app\common\controller\BaseController;
use app\common\model\UsedCar;
use app\common\model\UsedCarPdfLog;
use app\common\model\User;
@@ -9,8 +8,8 @@ use app\common\service\OssService;
use app\common\service\SparkService;
use app\common\traits\crud\CrudControllerTrait;
use app\common\service\UsedCarService;
use app\common\utils\crawl\service\UsedCarCrawlSingle;
use app\common\utils\pdf\service\MpdfDriver;
use app\queue\utils\XunfeiFanyi;
use support\exception\BusinessException;
use support\Request;
@@ -127,6 +126,41 @@ class UsedCarController extends BaseController
return $data;
}
public function crawlCar(Request $request)
{
try {
$url = $request->get('url');
if (empty($url)){
throw new BusinessException('url不能为空');
}
$info = (new UsedCarCrawlSingle())->crawlCar($url);
$data = [
'user_id' => $request->user['id'],
'product_name' => $info['title'],
'price' => $info['price']<200 ? $info['price']*10000 : $info['price'],
'source_place' => $info['source_place'],
'displacement' => $info['capacity'], //排量
'mileage' => $info['usage_mileage']*10000, //里程
'power' => $info['horsepower'], //马力
'body_color' => $info['body_color'],
'images' => $info['image'],
'image_pull' => $info['image_pull'],
'condition_desc' => $info['condition_desc'],
'attachment' => '',
'source_url' => $url,
'crawl_car_id' => $info['car_id'],
'crawl_brand_id' => $info['brand_id'],
'crawl_series_id' => $info['series_id'],
'used_car_id' => $info['used_car_id'],
];
return $this->success($data);
} catch (\Throwable $e) {
return $this->error($e->getMessage().'|'.$e->getLine().$e->getFile().$e->getTraceAsString());
}
}
public function crawlKeyword(Request $request){
try {
// $keyword = $request->get('keyword');
app/common/model/UsedCar.php
+1 -1
View File
@@ -40,7 +40,7 @@ class UsedCar extends Model
/**
* 允许自动填充模型字段
*/
protected $fillable = ['id','code', 'user_id', 'product_name', 'vin', 'price', 'source_place', 'plate_place', 'plate_time', 'displacement','mileage','power', 'body_color', 'images', 'condition_desc', 'attachment', 'crawl_title', 'crawl_brand', 'crawl_brand_id', 'crawl_series', 'crawl_series_id', 'crawl_year', 'crawl_msrp', 'crawl_car_id', 'status', 'sort', 'created_at', 'updated_at'];
protected $fillable = ['id','code', 'user_id', 'product_name', 'vin', 'price', 'source_place', 'plate_place', 'plate_time', 'displacement','mileage','power', 'body_color', 'images', 'condition_desc', 'attachment', 'pdf_src', 'source_url', 'crawl_title', 'crawl_brand', 'crawl_brand_id', 'crawl_series', 'crawl_series_id', 'crawl_year', 'crawl_msrp', 'crawl_car_id', 'status', 'sort', 'created_at', 'updated_at'];
/**
* 不允许自动填充模型字段
app/common/utils/crawl/service/UsedCarCrawlSingle.php
+433
View File
@@ -0,0 +1,433 @@
<?php
namespace app\common\utils\crawl\service;
use app\common\model\UsedcarCrawlResultsImages;
use app\common\service\UploadService;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException;
use QL\QueryList;
use support\exception\BusinessException;
use support\Log;
use Throwable;
class UsedCarCrawlSingle
{
// 默认请求头
private const DEFAULT_HEADERS = [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept-Language' => 'zh-CN,zh;q=0.9',
'Referer' => 'https://www.dongchedi.com/',
];
/**
* 获取网页HTML内容
*
* @param string $url
* @return string
* @throws GuzzleException
*/
public function fetchHtml(string $url): string
{
$client = new Client([
'verify' => false,
]);
$response = $client->request('GET', $url, [
'headers' => self::DEFAULT_HEADERS,
'timeout' => 10,
]);
return (string)$response->getBody();
}
public function crawlCar(string $url){
$html = $this->fetchHtml($url);
if ($html === '') {
throw new BusinessException("页面内容为空: {$url}");
}
$info = $this->parseUsedCarDetail($html, $url);
// $insertId = $this->saveUsedCarBasicInfo($info);
// 下载图片 + 上传 + 入库（图片失败不影响主记录）
$imagesArray = json_decode($info['images'] ?? '[]', true);
if (json_last_error() !== JSON_ERROR_NONE) {
$imagesArray = [];
}
$saveDir = public_path('uploads/usedcar/' . $info['used_car_id']);
$image = $this->uploadAndRemove($imagesArray, $saveDir);
$info['image'] = $image['image'];
$info['image_pull'] = $image['image_pull'];
// 继续派发配置页任务
// if (!empty($info['car_id_url'])) {
// $this->pushUrl($info['car_id_url'], 'config', $insertId);
// }
return $info;
}
/**
* 下载 -> 上传 -> 删除本地
*
* @param array $images 原始相册URL列表
* @param string $path 本地保存目录
*/
public function uploadAndRemove(array $images, string $path)
{
if (empty($images)) {
return false;
}
// 下载图片
$filePatch = $this->downloadImageAlbum($images, $path);
if (empty($filePatch)) {
return false;
}
$uploadService = new UploadService();
$image = [];
$image_pull = [];
foreach ($filePatch as $url => $filePath) {
if ($filePath === null) {
continue;
}
try {
// 上传云端
$param = [
'path' => $filePath,
'scene' => 'usedcar_crawl',
'category' => 'usedcar',
];
$result = $uploadService->uploadLocalFile($param);
if (empty($result['file_name'])) {
// 上传失败则跳过，但不抛异常影响主流程
continue;
}
// 保存图片信息
// UsedcarCrawlResultsImages::create([
// 'uc_id' => $id,
// 'used_car_id' => $usedCarId,
// 'original_name'=> $result['original_name'] ?? '',
// 'file_path' => $result['file_name'],
// ]);
$image[] = $result['file_name'];
$image_pull[] = $result['file_url'];
} catch (Throwable $e) {
Log::error('crawlUsedCarImage'.$e->getMessage().'|'.$e->getLine().$e->getFile());
// 单张图失败不影响其他
// 可以按需写日志
} finally {
// 删除本地文件
if (is_file($filePath)) {
@unlink($filePath);
}
}
}
// 删除目录
rmdir($path);
return ['image' => $image, 'image_pull' => $image_pull];
}
/**
* 批量下载相册
*
* @param array $urls 图片URL数组
* @param string $saveDir 保存目录
* @return array [原始URL => 保存路径或null]
*/
public function downloadImageAlbum(array $urls, string $saveDir): array
{
$result = [];
foreach ($urls as $url) {
$url = trim((string)$url);
if ($url === '') {
continue;
}
if (strpos($url, '//') === 0) {
$url = 'https:' . $url;
}
try {
$path = $this->downloadImage($url, $saveDir);
} catch (Throwable $e) {
$path = null;
}
$result[$url] = $path;
}
return $result;
}
/**
* 下载单张图片
*/
public function downloadImage(string $url, string $saveDir, ?string $filename = null, array $headers = []): ?string
{
if (!is_dir($saveDir)) {
if (!@mkdir($saveDir, 0777, true) && !is_dir($saveDir)) {
// echo "[Image] 创建目录失败：{$saveDir}\n";
return null;
}
}
if ($filename === null) {
$path = parse_url($url, PHP_URL_PATH);
$ext = pathinfo($path ?? '', PATHINFO_EXTENSION);
if ($ext === '') {
$ext = 'jpg';
}
$filename = md5($url) . '.' . $ext;
}
$savePath = rtrim($saveDir, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . $filename;
$defaultHeaders = [
'User-Agent: ' . self::DEFAULT_HEADERS['User-Agent'],
'Accept: image/avif,image/webp,image/apng,image/*,*/*;q=0.8',
'Referer: ' . self::DEFAULT_HEADERS['Referer'],
];
foreach ($headers as $h) {
$defaultHeaders[] = $h;
}
$ch = curl_init($url);
curl_setopt_array($ch, [
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_CONNECTTIMEOUT => 5,
CURLOPT_TIMEOUT => 15,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_SSL_VERIFYHOST => false,
CURLOPT_HTTPHEADER => $defaultHeaders,
]);
$data = curl_exec($ch);
$errNo = curl_errno($ch);
$code = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE);
curl_close($ch);
if ($errNo !== 0 || $code !== 200 || $data === false || $data === '') {
return null;
}
$bytes = @file_put_contents($savePath, $data);
if ($bytes === false) {
return null;
}
return $savePath;
}
/**
* 解析二手车详情页信息
*
* @param string $html
* @param string $url
* @return array
* @throws BusinessException
*/
public function parseUsedCarDetail(string $html = '', string $url = ''): array
{
try {
$ql = QueryList::html($html);
} catch (Throwable $e) {
throw new BusinessException('解析 HTML 失败（详情页）: ' . $e->getMessage());
}
$title = trim($ql->find('h1')->text());
$price = trim($ql->find('.head-info_price-wrap__Y4bxi > p')->text());
$pageText = $ql->find('body')->text();
$jsonText = $ql->find('#__NEXT_DATA__')->text();
if ($jsonText === '') {
throw new BusinessException('详情页未找到 __NEXT_DATA__ JSON');
}
$jsonArray = $this->decodeJson($jsonText, '详情页 __NEXT_DATA__');
// 车辆相册
$images = $jsonArray['props']['pageProps']['skuDetail']['head_images'] ?? [];
// 车辆ID / 年份
$carInfo = $jsonArray['props']['pageProps']['skuDetail']['car_info'] ?? [];
$carId = $carInfo['car_id'] ?? 0;
$year = (int)($carInfo['year'] ?? 0);
// 车辆ID URL
$carIdUrl = $carId ? ('https://www.dongchedi.com/auto/params-carIds-' . $carId) : '';
// 能源类型（两个字段择一）
$overview = $jsonArray['props']['pageProps']['skuDetail']['car_config_overview'] ?? [];
$fuelForm = $overview['power']['fuel_form'] ?? '';
$horsepower = $overview['power']['horsepower'] ?? ''; //马力
$capacity = $overview['power']['capacity'] ?? ''; //排量
$newFuelForm = $overview['new_energy_power']['fuel_form'] ?? '';
$resultFuelForm = $fuelForm === '' ? $newFuelForm : $fuelForm;
// 使用里程
$importantText = $jsonArray['props']['pageProps']['skuDetail']['important_text'] ?? '';
$importantTextArray = $importantText ? explode(' | ', $importantText) : [];
$usageMileage = '';
if (isset($importantTextArray[1])) {
$usageMileage = str_replace('万公里', '', $importantTextArray[1]);
}
// 车辆名称 品牌、车系、车型
$brandName = $carInfo['brand_name'] ?? '';
$brand_id = $carInfo['brand_id'] ?? '';
$seriesName = $carInfo['series_name'] ?? '';
$series_id = $carInfo['series_id'] ?? '';
$carName = $carInfo['car_name'] ?? '';
$bodyColor = $carInfo['body_color'] ?? ''; //颜色
// 商家信息
$shopInfo = $jsonArray['props']['pageProps']['skuDetail']['shop_info'] ?? [];
$formatShopInfo = [
'shop_id' => $shopInfo['shop_id'] ?? 0,
'shop_name' => $shopInfo['shop_short_name'] ?? '',
'shop_address' => $shopInfo['shop_address'] ?? '',
];
// 简介参数
$otherParams = $jsonArray['props']['pageProps']['skuDetail']['other_params'] ?? [];
$displacement = $this->getParamValue($otherParams, '排量');
$gearbox = $this->getParamValue($otherParams, '变速箱');
$sourcePlace = $this->getParamValue($otherParams, '车源地');
$formatCarInfo = [
'displacement' => $displacement,
'gearbox' => $gearbox,
];
// 提取新车指导价 / 比新车省金额
$newPrice = $this->extractPatternValue($pageText, '/新车指导价[:：]\s*([^\r\n万]*万)/u');
$priceDifference = $this->extractPatternValue($pageText, '/比新车省[:：]\s*([^\r\n万]*万)/u');
// 提取车况介绍
$conditionDesc = '';
if ($pageText && preg_match('/车况介绍\s*(.*?)\s*推荐理由/su', $pageText, $m)) {
$conditionDesc = trim($m[1]);
}
return [
'used_car_id' => $this->extractIdFromUrl($url),
'car_id' => $carId,
'url' => $url,
'title' => $title,
'price' => str_replace('万', '', $this->decodePrice($price)),
'new_price' => str_replace('万', '', $newPrice),
'price_difference' => str_replace('万', '', $priceDifference),
'condition_desc' => $conditionDesc,
'images' => json_encode($images, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
'car_id_url' => $carIdUrl,
'used_detail' => $jsonText,
'format_shop_info' => json_encode($formatShopInfo, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
'fuel_form' => $resultFuelForm,
'usage_mileage' => $usageMileage,
'brand_name' => $brandName,
'series_name' => $seriesName,
'car_name' => $carName,
'year' => $year,
'format_car_info' => json_encode($formatCarInfo, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
'body_color' => $bodyColor,
'horsepower' => $horsepower,
'capacity' => $capacity,
'source_place' => $sourcePlace,
'brand_id' => $brand_id,
'series_id' => $series_id,
];
}
/**
* 解码懂车帝价格
*/
protected function decodePrice(string $str): string
{
$map = [
"\u{E53D}" => '1',
"\u{E3F0}" => '2',
"\u{E422}" => '3',
"\u{E42C}" => '4',
"\u{E49C}" => '5',
"\u{E42B}" => '6',
"\u{E4FE}" => '7',
"\u{E548}" => '8',
"\u{E4C8}" => '9',
"\u{E453}" => '0',
"\u{E45F}" => '万',
];
return strtr($str, $map);
}
/**
* 从URL中提取ID
*/
protected function extractIdFromUrl(string $url)
{
if (preg_match('/\/(\d+)$/', $url, $matches)) {
return (int)$matches[1];
}
return false;
}
/**
* 提取符合正则表达式的值（只保留数字和小数点）
*/
protected function extractPatternValue(string $text, string $pattern): string
{
$value = '';
if ($text && preg_match($pattern, $text, $m)) {
$value = trim(preg_replace('/[^0-9.]/', '', $m[1]));
}
return $value;
}
/**
* 从参数列表中获取指定名称的值
*/
public function getParamValue(array $params, string $name): ?string
{
foreach ($params as $item) {
if (($item['name'] ?? null) === $name) {
return $item['value'] ?? null;
}
}
return null;
}
/**
* 安全解 JSON
*
* @param string $json
* @param string $context
* @return array
* @throws BusinessException
*/
protected function decodeJson(string $json, string $context = 'JSON'): array
{
$data = json_decode($json, true);
if (json_last_error() !== JSON_ERROR_NONE || !is_array($data)) {
throw new BusinessException($context . ' 解析失败：' . json_last_error_msg());
}
return $data;
}
}
config/route.php
+1
View File
@@ -67,6 +67,7 @@ Route::group('/common', function () {
Route::any('/update/status', [app\common\controller\UsedCarController::class, 'status']);
Route::any('/destroy', [app\common\controller\UsedCarController::class, 'destroy']);
Route::any('/crawl', [app\common\controller\UsedCarController::class, 'crawlKeyword']);
Route::any('/url_crawl', [app\common\controller\UsedCarController::class, 'crawlCar']);
});
Route::group('/user', function () {