Merge branch 'feature/sku_crawl' into develop

app/admin/service/BrandSeriesService.php

@@ -3,10 +3,19 @@
 namespace app\admin\service;
 
 use app\admin\validate\BrandSeriesValidate;
+use app\common\logic\Fingerprint;
+use app\common\logic\ModelNaming;
+use app\common\logic\SkuCodeBuilder;
 use app\common\model\Brand;
+use app\common\model\VehVehicleSku;
+use app\common\service\CrawlService;
+use app\common\service\VehVehicleSkuService;
 use app\common\traits\crud\CrudServiceTrait;
 use app\common\model\BrandSeries;
+use app\common\utils\CrudUtil;
+use support\Log;
 use support\Request;
+use Webman\RedisQueue\Redis;
 
 class BrandSeriesService
 {
@@ -123,4 +132,131 @@ class BrandSeriesService
         $data['series_code'] = $this->generateUniqueModificationCode($brand_id);
         return $data;
     }
+
+    /**
+     * 抓取车型数据
+     * @param Request $request
+     * @return array
+     * @throws \Exception
+     */
+    public function crawlModel(Request $request)
+    {
+        $series_id = $request->get('brand_series_id');
+
+        // 验证品牌是否存在
+        $series = BrandSeries::where('id', $series_id)->first();
+        if (!$series) {
+            throw new \Exception("系列ID {$series_id} 不存在");
+        }
+
+        // 获取品牌信息
+        $series = BrandSeries::where('id', $series_id)->first();
+        if (!$series) {
+            throw new \Exception("系列ID {$series_id} 不存在");
+        }
+
+        // 获取品牌信息
+        $brand = Brand::where('id', $series->brand_id)->first();
+
+        // 执行抓取
+        $crawlService = new CrawlService();
+        $models = $crawlService->seriesModel($brand->brand_name . ' ' . $series->series_name);
+
+        foreach ($models as $key => $model) {
+            $models[$key]['year'] = $this->extractYearFromCarModel($model['title']);
+        }
+
+        return $models;
+
+//        // 将抓取任务加入队列
+//        Redis::send('series-crawl', [
+//            'brand_series_id' => $series
+//        ]);
+//
+//        return true;
+    }
+
+    /**
+     * @param Request $request
+     * @return array|string
+     * @throws \Exception
+     */
+    public function crawlModelCreate(Request $request)
+    {
+        $series_id = $request->input('series_id');
+//        $brand_id = $request->input('brand_id');
+        $models = $request->input('models');
+
+        // 验证系列是否存在
+        $series = BrandSeries::where('id', $series_id)->first();
+        if (!$series) {
+            throw new \Exception("系列ID {$series_id} 不存在");
+        }
+        // 获取品牌信息
+        $brand = Brand::where('id', $series->brand_id)->first();
+
+        // 保存车型数据
+        $createdCount = 0;
+        $repeat = 0;
+        $fail = 0;
+        $msg = '';
+
+        foreach ($models as $item) {
+            $modelData = [
+                'brand_code' => $brand->brand_code ?? '',
+                'brand_name' => $brand->brand_name,
+                'series_code' => $series->series_code ?? '',
+                'series_name' => $series->series_name ?? '',
+                'model_name' => $item['title'],
+                'model_year' => $item['year'],
+                'msrp' => $item['msrp'],
+            ];
+            $modelData['model_code'] = ModelNaming::buildModelCode($modelData);
+            $modelData['sku_code'] = SkuCodeBuilder::build($modelData);
+            // 指纹
+            $modelData['fingerprint'] = Fingerprint::makeFromArray($modelData);
+
+            try {
+                $model = VehVehicleSku::where('brand_name', $brand->brand_name)->where('series_name', $series->series_name)->where('model_name', $item['title'])->first();
+                if (!empty($model)) {
+                    $repeat++;
+                    continue;
+                }
+                CrudUtil::create(VehVehicleSku::class, $modelData);
+                $createdCount++;
+            } catch (\Exception $e) {
+                // 记录错误但继续处理其他车型
+                $msg .= "创建车型失败: " . $e->getMessage() . "\n";
+                Log::error("创建车型失败:" . $msg, ['series_id' => $series_id, 'model' => $item]);
+                $fail++;
+            }
+        }
+
+        Log::info('series-crawl-' . $series_id . ' ' . $msg);
+        $msg = '创建车型成功' . $createdCount . '条';
+        $msg .= (!empty($repeat)) ? '，重复' . $repeat . '条' : '';
+        $msg .= (!empty($fail)) ? '，失败' . $fail . '条' : '';
+        return $msg;
+    }
+
+
+    /**
+     * 从车型名中提取年份
+     * @param string $carModel 车型名称
+     * @return string|null 提取到的年份或null
+     */
+    function extractYearFromCarModel($carModel)
+    {
+        // 匹配4位数字 + "款" 的模式
+        if (preg_match('/(\d{4})款/', $carModel, $matches)) {
+            return $matches[1];
+        }
+
+        // 匹配4位数字 + 空格或结尾的模式
+        if (preg_match('/^(\d{4})(?=\s|$)/', $carModel, $matches)) {
+            return $matches[1];
+        }
+
+        return null;
+    }
 }

app/admin/service/BrandService.php

@@ -3,9 +3,14 @@
 namespace app\admin\service;
 
 use app\admin\validate\BrandValidate;
+use app\common\model\BrandSeries;
+use app\common\service\CrawlService;
 use app\common\traits\crud\CrudServiceTrait;
 use app\common\model\Brand;
+use app\common\utils\CrudUtil;
+use support\Log;
 use support\Request;
+use Webman\RedisQueue\Redis;
 
 class BrandService
 {
@@ -77,4 +82,73 @@ class BrandService
 
         return $config;
     }
+
+    public function crawlSeries(Request $request)
+    {
+        $brand_id = $request->get('brand_id');
+
+        // 验证品牌是否存在
+        $brand = Brand::where('id', $brand_id)->first();
+        if (!$brand) {
+            throw new \Exception("品牌ID {$brand_id} 不存在");
+        }
+
+        // 获取品牌信息
+        try {
+            // 执行抓取
+            $crawlService = new CrawlService();
+            $series = $crawlService->brandSeries($brand->brand_name);
+
+            if (empty($series)) {
+                throw new \Exception("未抓取到品牌 {$brand->brand_name} 的系列数据");
+            }
+
+            return ['series' => $series];
+
+        } catch (\Throwable $th) {
+            Log::error($th);
+            throw $th;
+        }
+    }
+
+    public function crawlSeriesCreate(Request $request)
+    {
+        $brand_id = $request->input('brand_id');
+        $series = $request->input('series');
+
+        // 保存系列数据
+        $seriesService = new \app\admin\service\BrandSeriesService();
+        $createdCount = 0;
+        $repeat = 0;
+        $fail = 0;
+        $msg = '';
+
+        foreach ($series as $item) {
+            $seriesData = [
+                'brand_id' => $brand_id,
+                'series_code' => $seriesService->generateUniqueModificationCode($brand_id),
+                'series_name' => $item,
+            ];
+
+            try {
+                $brandseries = BrandSeries::where('brand_id', $brand_id)->where('series_name', $item)->first();
+                if (!empty($brandseries)) {
+                    $repeat++;
+                    continue;
+                }
+                CrudUtil::create(BrandSeries::class, $seriesData);
+                $createdCount++;
+
+            } catch (\Exception $e) {
+                // 记录错误但继续处理其他系列
+                $msg .= "创建系列失败: " . $e->getMessage() . "\n";
+                $fail++;
+            }
+        }
+        Log::info('brand-crawl-' . $brand_id . ' ' . $msg);
+        $msg = '创建系列成功' . $createdCount . '条';
+        $msg .= (!empty($repeat))?'，重复' . $repeat . '条':'';
+        $msg .= (!empty($fail))?'，失败' . $fail . '条':'';
+        return $msg;
+    }
 }

app/common/controller/BrandController.php

@@ -0,0 +1,90 @@
+<?php
+namespace app\common\controller;
+
+use app\common\controller\BaseController;
+use app\common\service\OssService;
+use app\common\traits\crud\CrudControllerTrait;
+use app\admin\service\BrandService;
+use support\Request;
+
+class BrandController extends BaseController
+{
+    use CrudControllerTrait;
+
+    protected $service;
+
+    public function __construct()
+    {
+        $this->service = new BrandService();
+    }
+
+    /**
+     * 列表页数据格式化
+     *
+     * @param array $data
+     * @return array
+     */
+     protected function indexFormat($data)
+     {
+         if (isset($data['created_at']))
+             $data['created_at'] = date('Y-m-d H:i:s', $data['created_at']);
+         if (isset($data['updated_at']))
+             $data['updated_at'] = date('Y-m-d H:i:s', $data['updated_at']);
+         if (!empty($data['logo']))
+             $data['logo_full'] = (new OssService)->generatePrivateUrl($data['logo']);
+         return $data;
+     }
+
+    /**
+     * 数据格式化
+     *
+     * @param array $data
+     * @return array
+     */
+    protected function showFormat($data)
+    {
+        if (isset($data['created_at']))
+            $data['created_at'] = date('Y-m-d H:i:s', $data['created_at']);
+        if (isset($data['updated_at']))
+            $data['updated_at'] = date('Y-m-d H:i:s', $data['updated_at']);
+        if (empty($data['logo']))
+            $data['logo_full'] = (new OssService)->generatePrivateUrl($data['logo']);
+        return $data;
+    }
+
+    /**
+     * 抓取品牌系列
+     * @param Request $request
+     * @return \support\Response
+     */
+    public function crawlSeries(Request $request){
+        try {
+            $result = $this->service->crawlSeries($request);
+            if($result){
+                return $this->success($result,'成功');
+            }else{
+                return $this->error('抓取任务提交失败');
+            }
+        } catch (\Exception $e) {
+            return $this->error($e->getMessage());
+        }
+    }
+
+    /**
+     * 创建被抓取且选中的品牌系列
+     * @param Request $request
+     * @return \support\Response
+     */
+    public function crawlSeriesCreate(Request $request){
+        try {
+            $result = $this->service->crawlSeriesCreate($request);
+            if($result){
+                return $this->success([],$result);
+            }else{
+                return $this->error('创建失败');
+            }
+        } catch (\Exception $e) {
+            return $this->error($e->getMessage());
+        }
+    }
+}

app/common/controller/BrandSeriesController.php

@@ -0,0 +1,85 @@
+<?php
+namespace app\common\controller;
+
+use app\common\controller\BaseController;
+use app\common\traits\crud\CrudControllerTrait;
+use app\admin\service\BrandSeriesService;
+use support\Request;
+
+class BrandSeriesController extends BaseController
+{
+    use CrudControllerTrait;
+
+    protected $service;
+
+    public function __construct()
+    {
+        $this->service = new BrandSeriesService();
+    }
+
+    /**
+     * 列表页数据格式化
+     *
+     * @param array $data
+     * @return array
+     */
+     protected function indexFormat($data)
+     {
+         if (isset($data['created_at']))
+             $data['created_at'] = date('Y-m-d H:i:s', $data['created_at']);
+         if (isset($data['updated_at']))
+             $data['updated_at'] = date('Y-m-d H:i:s', $data['updated_at']);
+         return $data;
+     }
+
+    /**
+     * 数据格式化
+     *
+     * @param array $data
+     * @return array
+     */
+    protected function showFormat($data)
+    {
+        if (isset($data['created_at']))
+            $data['created_at'] = date('Y-m-d H:i:s', $data['created_at']);
+        if (isset($data['updated_at']))
+            $data['updated_at'] = date('Y-m-d H:i:s', $data['updated_at']);
+        return $data;
+    }
+
+    /**
+     * 异步抓取系列车型
+     * @param Request $request
+     * @return \support\Response
+     */
+    public function crawlModels(Request $request){
+        try {
+            $result = $this->service->crawlModel($request);
+            if($result){
+                return $this->success($result,'抓取成功');
+            }else{
+                return $this->error('抓取任务提交失败');
+            }
+        } catch (\Exception $e) {
+            return $this->error($e->getMessage());
+        }
+    }
+
+    /**
+     * 异步抓取系列车型
+     * @param Request $request
+     * @return \support\Response
+     */
+    public function crawlModelCreate(Request $request){
+        try {
+            $result = $this->service->crawlModelCreate($request);
+            if($result){
+                return $this->success($result,'成功');
+            }else{
+                return $this->error('创建失败');
+            }
+        } catch (\Exception $e) {
+            return $this->error($e->getMessage());
+        }
+    }
+}

app/common/controller/SysPortController.php

@@ -0,0 +1,53 @@
+<?php
+namespace app\common\controller;
+
+use app\common\controller\BaseController;
+use app\common\model\Country;
+use app\common\traits\crud\CrudControllerTrait;
+use app\admin\service\SysPortService;
+
+class SysPortController extends BaseController
+{
+    use CrudControllerTrait;
+
+    protected $service;
+
+    public function __construct()
+    {
+        $this->service = new SysPortService();
+    }
+
+    /**
+     * 列表页数据格式化
+     *
+     * @param array $data
+     * @return array
+     */
+     protected function indexFormat($data)
+     {
+         if (isset($data['created_at']))
+             $data['created_at'] = date('Y-m-d H:i:s', $data['created_at']);
+         if (isset($data['updated_at']))
+             $data['updated_at'] = date('Y-m-d H:i:s', $data['updated_at']);
+         if (isset($data['country_id']))
+         if (isset($data['country_id']))
+            $data['country_name'] = Country::where('id', $data['country_id'])->value('cn_addr');
+         return $data;
+     }
+
+    /**
+     * 数据格式化
+     *
+     * @param array $data
+     * @return array
+     */
+    protected function showFormat($data)
+    {
+        if (isset($data['created_at']))
+            $data['created_at'] = date('Y-m-d H:i:s', $data['created_at']);
+        if (isset($data['updated_at']))
+            $data['updated_at'] = date('Y-m-d H:i:s', $data['updated_at']);
+        $data['country_name'] = Country::where('id', $data['country_id'])->value('cn_addr');
+        return $data;
+    }
+}

app/common/service/CrawlService.php

@@ -0,0 +1,76 @@
+<?php
+
+namespace app\common\service;
+
+use app\common\logic\BrandLogic;
+use QL\QueryList;
+
+class CrawlService
+{
+    public function brandSeries($brand)
+    {
+        $url = "https://www.dongchedi.com/search?currTab=6&keyword=".$brand;
+        $ql = QueryList::get($url );
+        $series = [];
+        $ql->find('div.card-name')->texts()->map(function($item) use(&$series) {
+            $series[] = $item;
+        });
+        return $series;
+    }
+
+    public function seriesModel($series)
+    {
+        $href = $this->modelSearch($series);
+
+        //例如 $seriesUrl = "https://www.dongchedi.com/auto/series/4865?anchorId=carModels"
+        if (empty($href)) {
+            return [];
+        }
+
+        $seriesUrl = (strpos($href, 'http') === 0) ? $href : ("https://www.dongchedi.com" . $href);
+        $ql = QueryList::get($seriesUrl);
+
+        $nodes = $ql->find('div.tw-grid div.tw-relative div.tw-items-center>a');
+        $texts = $nodes->texts()->all();
+        $links = $nodes->attrs('href')->all();
+        $nodes = $ql->find('div.tw-grid div.tw-text-color-gray-800');
+        $price = $nodes->texts()->all();
+
+        $results = [];
+        foreach ($texts as $index => $text) {
+            $link = $links[$index] ?? '';
+            if (!empty($link) && strpos($link, 'http') !== 0) {
+                $link = 'https://www.dongchedi.com' . $link;
+            }
+            $results[] = [
+                'title' => trim($text),
+                'link' => $link,
+                'msrp' => (int)(str_replace("万", "", $price[$index]) * 10000) ?? 0,
+            ];
+        }
+
+        return $results;
+    }
+
+//    public function modelDesc(array $models)
+//    {
+//
+//    }
+
+    /**
+     * 返回车系地址
+     * @param $series
+     * @return mixed|string
+     */
+    public function modelSearch($series)
+    {
+        $url = "https://www.dongchedi.com/search?keyword=".$series;
+        $ql = QueryList::get($url );
+        $href = '';
+        $ql->find('a.fold-btn')->attrs('href')
+            ->map(function ($item) use(&$href) {
+                $href = $item;
+            });
+        return $href;
+    }
+}

composer.json

@@ -46,7 +46,8 @@
     "phpoffice/phpspreadsheet": "^4.5",
     "phpmailer/phpmailer": "^6.10",
     "webman/gateway-worker": "^1.0",
-    "mpdf/mpdf": "^8.2"
+    "mpdf/mpdf": "^8.2",
+    "jaeger/querylist": "^4.4"
   },
   "suggest": {
     "ext-event": "For better performance. "

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "4914e483b0c6fc599378c91d2eab17cf",
+    "content-hash": "865e38bae84e7de0c95098cc87135440",
     "packages": [
         {
             "name": "aliyuncs/oss-sdk-php",
@@ -1803,6 +1803,161 @@
             },
             "time": "2025-06-06T18:15:26+00:00"
         },
+        {
+            "name": "jaeger/g-http",
+            "version": "V2.0.1",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/jae-jae/GHttp.git",
+                "reference": "29cb537c57f230980ca10622de190f04831e35e6"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/jae-jae/GHttp/zipball/29cb537c57f230980ca10622de190f04831e35e6",
+                "reference": "29cb537c57f230980ca10622de190f04831e35e6",
+                "shasum": ""
+            },
+            "require": {
+                "guzzlehttp/guzzle": "^6.0 || ^7.0",
+                "php": ">=8.1",
+                "symfony/cache": "^6.4 || ^7.0"
+            },
+            "require-dev": {
+                "predis/predis": "^2.2"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Jaeger\\": "src"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Jaeger",
+                    "email": "JaegerCode@gmail.com"
+                }
+            ],
+            "description": "Simple Http client base on GuzzleHttp",
+            "support": {
+                "issues": "https://github.com/jae-jae/GHttp/issues",
+                "source": "https://github.com/jae-jae/GHttp/tree/V2.0.1"
+            },
+            "time": "2024-06-14T08:13:32+00:00"
+        },
+        {
+            "name": "jaeger/phpquery-single",
+            "version": "1.1.1",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/jae-jae/phpQuery-single.git",
+                "reference": "39a650ade692a6b480c22220dce0c198d6a946fb"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/jae-jae/phpQuery-single/zipball/39a650ade692a6b480c22220dce0c198d6a946fb",
+                "reference": "39a650ade692a6b480c22220dce0c198d6a946fb",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=5.3.0"
+            },
+            "type": "library",
+            "autoload": {
+                "classmap": [
+                    "phpQuery.php"
+                ]
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Tobiasz Cudnik",
+                    "email": "tobiasz.cudnik@gmail.com",
+                    "homepage": "https://github.com/TobiaszCudnik",
+                    "role": "Developer"
+                },
+                {
+                    "name": "Jaeger",
+                    "role": "Packager"
+                }
+            ],
+            "description": "phpQuery单文件版本，是Querylist的依赖(http://querylist.cc/)，phpQuery项目主页:http://code.google.com/p/phpquery/",
+            "homepage": "http://code.google.com/p/phpquery/",
+            "support": {
+                "issues": "https://github.com/jae-jae/phpQuery-single/issues",
+                "source": "https://github.com/jae-jae/phpQuery-single/tree/1.1.1"
+            },
+            "time": "2022-03-26T15:01:16+00:00"
+        },
+        {
+            "name": "jaeger/querylist",
+            "version": "V4.4.6",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/jae-jae/QueryList.git",
+                "reference": "939f4e41b94b4d3ef77071fb7f386abd2ceeb59a"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/jae-jae/QueryList/zipball/939f4e41b94b4d3ef77071fb7f386abd2ceeb59a",
+                "reference": "939f4e41b94b4d3ef77071fb7f386abd2ceeb59a",
+                "shasum": ""
+            },
+            "require": {
+                "ext-dom": "*",
+                "jaeger/g-http": "^2.0",
+                "jaeger/phpquery-single": "^1",
+                "php": ">=8.1",
+                "symfony/var-dumper": ">3.4"
+            },
+            "require-dev": {
+                "phpunit/phpunit": "^8.5"
+            },
+            "type": "library",
+            "autoload": {
+                "files": [
+                    "src/Collect/Support/helpers.php",
+                    "src/Collect/Support/alias.php"
+                ],
+                "psr-4": {
+                    "QL\\": "src"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Jaeger",
+                    "email": "JaegerCode@gmail.com"
+                }
+            ],
+            "description": "Simple, elegant, extensible PHP Web Scraper (crawler/spider),Use the css3 dom selector,Based on phpQuery! 简洁、优雅、可扩展的PHP采集工具(爬虫)，基于phpQuery。",
+            "homepage": "http://querylist.cc",
+            "keywords": [
+                "QueryList",
+                "phpQuery",
+                "spider"
+            ],
+            "support": {
+                "issues": "https://github.com/jae-jae/QueryList/issues",
+                "source": "https://github.com/jae-jae/QueryList/tree/V4.4.6"
+            },
+            "funding": [
+                {
+                    "url": "https://opencollective.com/querylist",
+                    "type": "open_collective"
+                }
+            ],
+            "time": "2024-12-13T06:16:14+00:00"
+        },
         {
             "name": "laravel/serializable-closure",
             "version": "v2.0.4",

config/route.php

@@ -87,6 +87,38 @@ Route::group('/common', function () {
 
         Route::any('/unread', [app\common\controller\MessageUserStatusController::class, 'listUnRead']);
     });
+
+    // 品牌
+    Route::group('/brand', function () {
+        Route::any('/list', [app\common\controller\BrandController::class, 'index']);
+        Route::any('/show', [app\common\controller\BrandController::class, 'show']);
+        Route::any('/store', [app\common\controller\BrandController::class, 'store']);
+        Route::any('/update', [app\common\controller\BrandController::class, 'update']);
+        Route::any('/hot', [app\common\controller\BrandController::class, 'update']);
+        Route::any('/destroy', [app\common\controller\BrandController::class, 'destroy']);
+        Route::any('/crawlSeries', [app\common\controller\BrandController::class, 'crawlSeries']);
+        Route::any('/crawlSeriesCreate', [app\common\controller\BrandController::class, 'crawlSeriesCreate']);
+        // 车系
+        Route::group('/series', function () {
+            Route::any('/list', [app\common\controller\BrandSeriesController::class, 'index']);
+            Route::any('/show', [app\common\controller\BrandSeriesController::class, 'show']);
+            Route::any('/store', [app\common\controller\BrandSeriesController::class, 'store']);
+            Route::any('/update', [app\common\controller\BrandSeriesController::class, 'update']);
+            Route::any('/destroy', [app\common\controller\BrandSeriesController::class, 'destroy']);
+            Route::any('/crawlModels', [app\common\controller\BrandSeriesController::class, 'crawlModels']);
+            Route::any('/crawlModelCreate', [app\common\controller\BrandSeriesController::class, 'crawlModelCreate']);
+        });
+
+        // 港口
+        Route::group('/port', function () {
+            Route::any('/list', [app\common\controller\SysPortController::class, 'index']);
+            Route::any('/show', [app\common\controller\SysPortController::class, 'show']);
+            Route::any('/store', [app\common\controller\SysPortController::class, 'store']);
+            Route::any('/update', [app\common\controller\SysPortController::class, 'update']);
+            Route::any('/destroy', [app\common\controller\SysPortController::class, 'destroy']);
+        });
+    });
+    });
 });
 
 // 后台管理