车辆品牌与抓取车系

This commit is contained in:
2025-10-10 16:48:33 +08:00
parent 3091944e6a
commit 4fe008354d
9 changed files with 711 additions and 9 deletions
+140 -4
View File
@@ -3,10 +3,19 @@
namespace app\admin\service;
use app\admin\validate\BrandSeriesValidate;
use app\common\logic\Fingerprint;
use app\common\logic\ModelNaming;
use app\common\logic\SkuCodeBuilder;
use app\common\model\Brand;
use app\common\model\VehVehicleSku;
use app\common\service\CrawlService;
use app\common\service\VehVehicleSkuService;
use app\common\traits\crud\CrudServiceTrait;
use app\common\model\BrandSeries;
use app\common\utils\CrudUtil;
use support\Log;
use support\Request;
use Webman\RedisQueue\Redis;
class BrandSeriesService
{
@@ -28,7 +37,7 @@ class BrandSeriesService
*/
public function getValidator()
{
return new $this->validator;
return new $this->validator;
}
/**
@@ -50,13 +59,13 @@ class BrandSeriesService
// ====== ====== 筛选 ====== ======
$config['where']['brand_id'] = $p['brand_id'];
// 状态
if (isset($p['status']) && in_array($p['status'], [0,1])) {
if (isset($p['status']) && in_array($p['status'], [0, 1])) {
$config['where']['status'] = $p['status'];
}
// 关键字
if (isset($p['keyword']) && $p['keyword']) {
$k = '%' . $p['keyword'] . '%';
foreach (['series_code','series_name'] as $col) {
foreach (['series_code', 'series_name'] as $col) {
$config['orWhere'][] = [$col, 'like', $k];
}
}
@@ -92,7 +101,7 @@ class BrandSeriesService
$maxAttempts = 10; // 防止无限循环
$brand_code = Brand::where('id', $brand_id)->value('brand_code');
$brand_code = $brand_code??'BS';
$brand_code = $brand_code ?? 'BS';
for ($attempt = 0; $attempt < $maxAttempts; $attempt++) {
// 生成后5位随机字符
$suffix = '';
@@ -123,4 +132,131 @@ class BrandSeriesService
$data['series_code'] = $this->generateUniqueModificationCode($brand_id);
return $data;
}
/**
* 抓取车型数据
* @param Request $request
* @return array
* @throws \Exception
*/
public function crawlModel(Request $request)
{
$series_id = $request->get('brand_series_id');
// 验证品牌是否存在
$series = BrandSeries::where('id', $series_id)->first();
if (!$series) {
throw new \Exception("系列ID {$series_id} 不存在");
}
// 获取品牌信息
$series = BrandSeries::where('id', $series_id)->first();
if (!$series) {
throw new \Exception("系列ID {$series_id} 不存在");
}
// 获取品牌信息
$brand = Brand::where('id', $series->brand_id)->first();
// 执行抓取
$crawlService = new CrawlService();
$models = $crawlService->seriesModel($brand->brand_name . ' ' . $series->series_name);
foreach ($models as $key => $model) {
$models[$key]['year'] = $this->extractYearFromCarModel($model['title']);
}
return $models;
// // 将抓取任务加入队列
// Redis::send('series-crawl', [
// 'brand_series_id' => $series
// ]);
//
// return true;
}
/**
* @param Request $request
* @return array|string
* @throws \Exception
*/
public function crawlModelCreate(Request $request)
{
$series_id = $request->input('series_id');
// $brand_id = $request->input('brand_id');
$models = $request->input('models');
// 验证系列是否存在
$series = BrandSeries::where('id', $series_id)->first();
if (!$series) {
throw new \Exception("系列ID {$series_id} 不存在");
}
// 获取品牌信息
$brand = Brand::where('id', $series->brand_id)->first();
// 保存车型数据
$createdCount = 0;
$repeat = 0;
$fail = 0;
$msg = '';
foreach ($models as $item) {
$modelData = [
'brand_code' => $brand->brand_code ?? '',
'brand_name' => $brand->brand_name,
'series_code' => $series->series_code ?? '',
'series_name' => $series->series_name ?? '',
'model_name' => $item['title'],
'model_year' => $item['year'],
'msrp' => $item['msrp'],
];
$modelData['model_code'] = ModelNaming::buildModelCode($modelData);
$modelData['sku_code'] = SkuCodeBuilder::build($modelData);
// 指纹
$modelData['fingerprint'] = Fingerprint::makeFromArray($modelData);
try {
$model = VehVehicleSku::where('brand_name', $brand->brand_name)->where('series_name', $series->series_name)->where('model_name', $item['title'])->first();
if (!empty($model)) {
$repeat++;
continue;
}
CrudUtil::create(VehVehicleSku::class, $modelData);
$createdCount++;
} catch (\Exception $e) {
// 记录错误但继续处理其他车型
$msg .= "创建车型失败: " . $e->getMessage() . "\n";
Log::error("创建车型失败:" . $msg, ['series_id' => $series_id, 'model' => $item]);
$fail++;
}
}
Log::info('series-crawl-' . $series_id . ' ' . $msg);
$msg = '创建车型成功' . $createdCount . '条';
$msg .= (!empty($repeat)) ? ',重复' . $repeat . '条' : '';
$msg .= (!empty($fail)) ? ',失败' . $fail . '条' : '';
return $msg;
}
/**
* 从车型名中提取年份
* @param string $carModel 车型名称
* @return string|null 提取到的年份或null
*/
function extractYearFromCarModel($carModel)
{
// 匹配4位数字 + "款" 的模式
if (preg_match('/(\d{4})款/', $carModel, $matches)) {
return $matches[1];
}
// 匹配4位数字 + 空格或结尾的模式
if (preg_match('/^(\d{4})(?=\s|$)/', $carModel, $matches)) {
return $matches[1];
}
return null;
}
}
+77 -3
View File
@@ -3,9 +3,14 @@
namespace app\admin\service;
use app\admin\validate\BrandValidate;
use app\common\model\BrandSeries;
use app\common\service\CrawlService;
use app\common\traits\crud\CrudServiceTrait;
use app\common\model\Brand;
use app\common\utils\CrudUtil;
use support\Log;
use support\Request;
use Webman\RedisQueue\Redis;
class BrandService
{
@@ -27,7 +32,7 @@ class BrandService
*/
public function getValidator()
{
return new $this->validator;
return new $this->validator;
}
/**
@@ -49,13 +54,13 @@ class BrandService
// ====== ====== 筛选 ====== ======
// 状态
if (isset($p['status']) && in_array($p['status'], [0,1])) {
if (isset($p['status']) && in_array($p['status'], [0, 1])) {
$config['where']['status'] = $p['status'];
}
// 关键字
if (isset($p['keyword']) && $p['keyword']) {
$k = '%' . $p['keyword'] . '%';
foreach (['brand_code','brand_name'] as $col) {
foreach (['brand_code', 'brand_name'] as $col) {
$config['orWhere'][] = [$col, 'like', $k];
}
}
@@ -77,4 +82,73 @@ class BrandService
return $config;
}
public function crawlSeries(Request $request)
{
$brand_id = $request->get('brand_id');
// 验证品牌是否存在
$brand = Brand::where('id', $brand_id)->first();
if (!$brand) {
throw new \Exception("品牌ID {$brand_id} 不存在");
}
// 获取品牌信息
try {
// 执行抓取
$crawlService = new CrawlService();
$series = $crawlService->brandSeries($brand->brand_name);
if (empty($series)) {
throw new \Exception("未抓取到品牌 {$brand->brand_name} 的系列数据");
}
return ['series' => $series];
} catch (\Throwable $th) {
Log::error($th);
throw $th;
}
}
public function crawlSeriesCreate(Request $request)
{
$brand_id = $request->input('brand_id');
$series = $request->input('series');
// 保存系列数据
$seriesService = new \app\admin\service\BrandSeriesService();
$createdCount = 0;
$repeat = 0;
$fail = 0;
$msg = '';
foreach ($series as $item) {
$seriesData = [
'brand_id' => $brand_id,
'series_code' => $seriesService->generateUniqueModificationCode($brand_id),
'series_name' => $item,
];
try {
$brandseries = BrandSeries::where('brand_id', $brand_id)->where('series_name', $item)->first();
if (!empty($brandseries)) {
$repeat++;
continue;
}
CrudUtil::create(BrandSeries::class, $seriesData);
$createdCount++;
} catch (\Exception $e) {
// 记录错误但继续处理其他系列
$msg .= "创建系列失败: " . $e->getMessage() . "\n";
$fail++;
}
}
Log::info('brand-crawl-' . $brand_id . ' ' . $msg);
$msg = '创建系列成功' . $createdCount . '条';
$msg .= (!empty($repeat))?',重复' . $repeat . '条':'';
$msg .= (!empty($fail))?',失败' . $fail . '条':'';
return $msg;
}
}
+90
View File
@@ -0,0 +1,90 @@
<?php
namespace app\common\controller;
use app\common\controller\BaseController;
use app\common\service\OssService;
use app\common\traits\crud\CrudControllerTrait;
use app\admin\service\BrandService;
use support\Request;
class BrandController extends BaseController
{
use CrudControllerTrait;
protected $service;
public function __construct()
{
$this->service = new BrandService();
}
/**
* 列表页数据格式化
*
* @param array $data
* @return array
*/
protected function indexFormat($data)
{
if (isset($data['created_at']))
$data['created_at'] = date('Y-m-d H:i:s', $data['created_at']);
if (isset($data['updated_at']))
$data['updated_at'] = date('Y-m-d H:i:s', $data['updated_at']);
if (!empty($data['logo']))
$data['logo_full'] = (new OssService)->generatePrivateUrl($data['logo']);
return $data;
}
/**
* 数据格式化
*
* @param array $data
* @return array
*/
protected function showFormat($data)
{
if (isset($data['created_at']))
$data['created_at'] = date('Y-m-d H:i:s', $data['created_at']);
if (isset($data['updated_at']))
$data['updated_at'] = date('Y-m-d H:i:s', $data['updated_at']);
if (empty($data['logo']))
$data['logo_full'] = (new OssService)->generatePrivateUrl($data['logo']);
return $data;
}
/**
* 抓取品牌系列
* @param Request $request
* @return \support\Response
*/
public function crawlSeries(Request $request){
try {
$result = $this->service->crawlSeries($request);
if($result){
return $this->success($result,'成功');
}else{
return $this->error('抓取任务提交失败');
}
} catch (\Exception $e) {
return $this->error($e->getMessage());
}
}
/**
* 创建被抓取且选中的品牌系列
* @param Request $request
* @return \support\Response
*/
public function crawlSeriesCreate(Request $request){
try {
$result = $this->service->crawlSeriesCreate($request);
if($result){
return $this->success([],$result);
}else{
return $this->error('创建失败');
}
} catch (\Exception $e) {
return $this->error($e->getMessage());
}
}
}
@@ -0,0 +1,85 @@
<?php
namespace app\common\controller;
use app\common\controller\BaseController;
use app\common\traits\crud\CrudControllerTrait;
use app\admin\service\BrandSeriesService;
use support\Request;
class BrandSeriesController extends BaseController
{
use CrudControllerTrait;
protected $service;
public function __construct()
{
$this->service = new BrandSeriesService();
}
/**
* 列表页数据格式化
*
* @param array $data
* @return array
*/
protected function indexFormat($data)
{
if (isset($data['created_at']))
$data['created_at'] = date('Y-m-d H:i:s', $data['created_at']);
if (isset($data['updated_at']))
$data['updated_at'] = date('Y-m-d H:i:s', $data['updated_at']);
return $data;
}
/**
* 数据格式化
*
* @param array $data
* @return array
*/
protected function showFormat($data)
{
if (isset($data['created_at']))
$data['created_at'] = date('Y-m-d H:i:s', $data['created_at']);
if (isset($data['updated_at']))
$data['updated_at'] = date('Y-m-d H:i:s', $data['updated_at']);
return $data;
}
/**
* 异步抓取系列车型
* @param Request $request
* @return \support\Response
*/
public function crawlModels(Request $request){
try {
$result = $this->service->crawlModel($request);
if($result){
return $this->success($result,'抓取成功');
}else{
return $this->error('抓取任务提交失败');
}
} catch (\Exception $e) {
return $this->error($e->getMessage());
}
}
/**
* 异步抓取系列车型
* @param Request $request
* @return \support\Response
*/
public function crawlModelCreate(Request $request){
try {
$result = $this->service->crawlModelCreate($request);
if($result){
return $this->success($result,'成功');
}else{
return $this->error('创建失败');
}
} catch (\Exception $e) {
return $this->error($e->getMessage());
}
}
}
@@ -0,0 +1,53 @@
<?php
namespace app\common\controller;
use app\common\controller\BaseController;
use app\common\model\Country;
use app\common\traits\crud\CrudControllerTrait;
use app\admin\service\SysPortService;
class SysPortController extends BaseController
{
use CrudControllerTrait;
protected $service;
public function __construct()
{
$this->service = new SysPortService();
}
/**
* 列表页数据格式化
*
* @param array $data
* @return array
*/
protected function indexFormat($data)
{
if (isset($data['created_at']))
$data['created_at'] = date('Y-m-d H:i:s', $data['created_at']);
if (isset($data['updated_at']))
$data['updated_at'] = date('Y-m-d H:i:s', $data['updated_at']);
if (isset($data['country_id']))
if (isset($data['country_id']))
$data['country_name'] = Country::where('id', $data['country_id'])->value('cn_addr');
return $data;
}
/**
* 数据格式化
*
* @param array $data
* @return array
*/
protected function showFormat($data)
{
if (isset($data['created_at']))
$data['created_at'] = date('Y-m-d H:i:s', $data['created_at']);
if (isset($data['updated_at']))
$data['updated_at'] = date('Y-m-d H:i:s', $data['updated_at']);
$data['country_name'] = Country::where('id', $data['country_id'])->value('cn_addr');
return $data;
}
}
+76
View File
@@ -0,0 +1,76 @@
<?php
namespace app\common\service;
use app\common\logic\BrandLogic;
use QL\QueryList;
class CrawlService
{
public function brandSeries($brand)
{
$url = "https://www.dongchedi.com/search?currTab=6&keyword=".$brand;
$ql = QueryList::get($url );
$series = [];
$ql->find('div.card-name')->texts()->map(function($item) use(&$series) {
$series[] = $item;
});
return $series;
}
public function seriesModel($series)
{
$href = $this->modelSearch($series);
//例如 $seriesUrl = "https://www.dongchedi.com/auto/series/4865?anchorId=carModels"
if (empty($href)) {
return [];
}
$seriesUrl = (strpos($href, 'http') === 0) ? $href : ("https://www.dongchedi.com" . $href);
$ql = QueryList::get($seriesUrl);
$nodes = $ql->find('div.tw-grid div.tw-relative div.tw-items-center>a');
$texts = $nodes->texts()->all();
$links = $nodes->attrs('href')->all();
$nodes = $ql->find('div.tw-grid div.tw-text-color-gray-800');
$price = $nodes->texts()->all();
$results = [];
foreach ($texts as $index => $text) {
$link = $links[$index] ?? '';
if (!empty($link) && strpos($link, 'http') !== 0) {
$link = 'https://www.dongchedi.com' . $link;
}
$results[] = [
'title' => trim($text),
'link' => $link,
'msrp' => (int)(str_replace("万", "", $price[$index]) * 10000) ?? 0,
];
}
return $results;
}
// public function modelDesc(array $models)
// {
//
// }
/**
* 返回车系地址
* @param $series
* @return mixed|string
*/
public function modelSearch($series)
{
$url = "https://www.dongchedi.com/search?keyword=".$series;
$ql = QueryList::get($url );
$href = '';
$ql->find('a.fold-btn')->attrs('href')
->map(function ($item) use(&$href) {
$href = $item;
});
return $href;
}
}
+2 -1
View File
@@ -46,7 +46,8 @@
"phpoffice/phpspreadsheet": "^4.5",
"phpmailer/phpmailer": "^6.10",
"webman/gateway-worker": "^1.0",
"mpdf/mpdf": "^8.2"
"mpdf/mpdf": "^8.2",
"jaeger/querylist": "^4.4"
},
"suggest": {
"ext-event": "For better performance. "
Generated
+156 -1
View File
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "4914e483b0c6fc599378c91d2eab17cf",
"content-hash": "865e38bae84e7de0c95098cc87135440",
"packages": [
{
"name": "aliyuncs/oss-sdk-php",
@@ -1803,6 +1803,161 @@
},
"time": "2025-06-06T18:15:26+00:00"
},
{
"name": "jaeger/g-http",
"version": "V2.0.1",
"source": {
"type": "git",
"url": "https://github.com/jae-jae/GHttp.git",
"reference": "29cb537c57f230980ca10622de190f04831e35e6"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/jae-jae/GHttp/zipball/29cb537c57f230980ca10622de190f04831e35e6",
"reference": "29cb537c57f230980ca10622de190f04831e35e6",
"shasum": ""
},
"require": {
"guzzlehttp/guzzle": "^6.0 || ^7.0",
"php": ">=8.1",
"symfony/cache": "^6.4 || ^7.0"
},
"require-dev": {
"predis/predis": "^2.2"
},
"type": "library",
"autoload": {
"psr-4": {
"Jaeger\\": "src"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Jaeger",
"email": "JaegerCode@gmail.com"
}
],
"description": "Simple Http client base on GuzzleHttp",
"support": {
"issues": "https://github.com/jae-jae/GHttp/issues",
"source": "https://github.com/jae-jae/GHttp/tree/V2.0.1"
},
"time": "2024-06-14T08:13:32+00:00"
},
{
"name": "jaeger/phpquery-single",
"version": "1.1.1",
"source": {
"type": "git",
"url": "https://github.com/jae-jae/phpQuery-single.git",
"reference": "39a650ade692a6b480c22220dce0c198d6a946fb"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/jae-jae/phpQuery-single/zipball/39a650ade692a6b480c22220dce0c198d6a946fb",
"reference": "39a650ade692a6b480c22220dce0c198d6a946fb",
"shasum": ""
},
"require": {
"php": ">=5.3.0"
},
"type": "library",
"autoload": {
"classmap": [
"phpQuery.php"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Tobiasz Cudnik",
"email": "tobiasz.cudnik@gmail.com",
"homepage": "https://github.com/TobiaszCudnik",
"role": "Developer"
},
{
"name": "Jaeger",
"role": "Packager"
}
],
"description": "phpQuery单文件版本,是Querylist的依赖(http://querylist.cc/),phpQuery项目主页:http://code.google.com/p/phpquery/",
"homepage": "http://code.google.com/p/phpquery/",
"support": {
"issues": "https://github.com/jae-jae/phpQuery-single/issues",
"source": "https://github.com/jae-jae/phpQuery-single/tree/1.1.1"
},
"time": "2022-03-26T15:01:16+00:00"
},
{
"name": "jaeger/querylist",
"version": "V4.4.6",
"source": {
"type": "git",
"url": "https://github.com/jae-jae/QueryList.git",
"reference": "939f4e41b94b4d3ef77071fb7f386abd2ceeb59a"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/jae-jae/QueryList/zipball/939f4e41b94b4d3ef77071fb7f386abd2ceeb59a",
"reference": "939f4e41b94b4d3ef77071fb7f386abd2ceeb59a",
"shasum": ""
},
"require": {
"ext-dom": "*",
"jaeger/g-http": "^2.0",
"jaeger/phpquery-single": "^1",
"php": ">=8.1",
"symfony/var-dumper": ">3.4"
},
"require-dev": {
"phpunit/phpunit": "^8.5"
},
"type": "library",
"autoload": {
"files": [
"src/Collect/Support/helpers.php",
"src/Collect/Support/alias.php"
],
"psr-4": {
"QL\\": "src"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Jaeger",
"email": "JaegerCode@gmail.com"
}
],
"description": "Simple, elegant, extensible PHP Web Scraper (crawler/spider),Use the css3 dom selector,Based on phpQuery! 简洁、优雅、可扩展的PHP采集工具(爬虫),基于phpQuery。",
"homepage": "http://querylist.cc",
"keywords": [
"QueryList",
"phpQuery",
"spider"
],
"support": {
"issues": "https://github.com/jae-jae/QueryList/issues",
"source": "https://github.com/jae-jae/QueryList/tree/V4.4.6"
},
"funding": [
{
"url": "https://opencollective.com/querylist",
"type": "open_collective"
}
],
"time": "2024-12-13T06:16:14+00:00"
},
{
"name": "laravel/serializable-closure",
"version": "v2.0.4",
+32
View File
@@ -87,6 +87,38 @@ Route::group('/common', function () {
Route::any('/unread', [app\common\controller\MessageUserStatusController::class, 'listUnRead']);
});
// 品牌
Route::group('/brand', function () {
Route::any('/list', [app\common\controller\BrandController::class, 'index']);
Route::any('/show', [app\common\controller\BrandController::class, 'show']);
Route::any('/store', [app\common\controller\BrandController::class, 'store']);
Route::any('/update', [app\common\controller\BrandController::class, 'update']);
Route::any('/hot', [app\common\controller\BrandController::class, 'update']);
Route::any('/destroy', [app\common\controller\BrandController::class, 'destroy']);
Route::any('/crawlSeries', [app\common\controller\BrandController::class, 'crawlSeries']);
Route::any('/crawlSeriesCreate', [app\common\controller\BrandController::class, 'crawlSeriesCreate']);
// 车系
Route::group('/series', function () {
Route::any('/list', [app\common\controller\BrandSeriesController::class, 'index']);
Route::any('/show', [app\common\controller\BrandSeriesController::class, 'show']);
Route::any('/store', [app\common\controller\BrandSeriesController::class, 'store']);
Route::any('/update', [app\common\controller\BrandSeriesController::class, 'update']);
Route::any('/destroy', [app\common\controller\BrandSeriesController::class, 'destroy']);
Route::any('/crawlModels', [app\common\controller\BrandSeriesController::class, 'crawlModels']);
Route::any('/crawlModelCreate', [app\common\controller\BrandSeriesController::class, 'crawlModelCreate']);
});
// 港口
Route::group('/port', function () {
Route::any('/list', [app\common\controller\SysPortController::class, 'index']);
Route::any('/show', [app\common\controller\SysPortController::class, 'show']);
Route::any('/store', [app\common\controller\SysPortController::class, 'store']);
Route::any('/update', [app\common\controller\SysPortController::class, 'update']);
Route::any('/destroy', [app\common\controller\SysPortController::class, 'destroy']);
});
});
});
});
// 后台管理