汽车之家链接转换,抓取优化
This commit is contained in:
@@ -7,6 +7,7 @@ use app\common\validate\UsedCarValidate;
|
|||||||
use app\common\traits\crud\CrudServiceTrait;
|
use app\common\traits\crud\CrudServiceTrait;
|
||||||
use app\common\model\UsedCar;
|
use app\common\model\UsedCar;
|
||||||
use GuzzleHttp\Client;
|
use GuzzleHttp\Client;
|
||||||
|
use GuzzleHttp\Exception\GuzzleException;
|
||||||
use support\exception\BusinessException;
|
use support\exception\BusinessException;
|
||||||
use support\Log;
|
use support\Log;
|
||||||
use support\Request;
|
use support\Request;
|
||||||
@@ -450,18 +451,18 @@ class UsedCarService
|
|||||||
case str_contains($url,'www.che168.com/dealer'): //汽车之家
|
case str_contains($url,'www.che168.com/dealer'): //汽车之家
|
||||||
$info = $usedCarCrawl->crawlChe168Car($url);
|
$info = $usedCarCrawl->crawlChe168Car($url);
|
||||||
break;
|
break;
|
||||||
|
case str_contains($url,'www.che168.com/home?infoid='): //汽车之家pc端链接转换
|
||||||
|
$url = $usedCarCrawl->getChe168UrlByPc($url);
|
||||||
|
$info = $usedCarCrawl->crawlChe168Car($url);
|
||||||
|
break;
|
||||||
case str_contains($url,'wap.che168.com/dealer'): //汽车之家移动端链接
|
case str_contains($url,'wap.che168.com/dealer'): //汽车之家移动端链接
|
||||||
$finalUrl = $usedCarCrawl->getChe168UrlByMobile($url);
|
$finalUrl = $usedCarCrawl->getChe168UrlByMobile($url);
|
||||||
$info = $usedCarCrawl->crawlChe168Car($finalUrl);
|
$info = $usedCarCrawl->crawlChe168Car($finalUrl);
|
||||||
break;
|
break;
|
||||||
case str_contains($url,'comm.app.autohome.com.cn'): //汽车之家移动端链接
|
case str_contains($url,'comm.app.autohome.com.cn'): //汽车之家移动端链接
|
||||||
$finalUrl = $usedCarCrawl->get168FinaUrl($url);
|
$finalUrl = $usedCarCrawl->get168FinaUrl($url);
|
||||||
var_dump($finalUrl);
|
|
||||||
$info = $usedCarCrawl->crawlChe168Car($finalUrl);
|
$info = $usedCarCrawl->crawlChe168Car($finalUrl);
|
||||||
break;
|
break;
|
||||||
case str_contains($url,'www.che168.com/dealer'): //汽车之家
|
|
||||||
$info = $usedCarCrawl->crawlChe168Car($url);
|
|
||||||
break;
|
|
||||||
default:
|
default:
|
||||||
Log::info('未知链接:'.$url);
|
Log::info('未知链接:'.$url);
|
||||||
throw new \Exception('加紧开发中');
|
throw new \Exception('加紧开发中');
|
||||||
|
|||||||
@@ -118,6 +118,28 @@ class UsedCarCrawlSingle
|
|||||||
return $pcUrl;
|
return $pcUrl;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 根据汽车之家pc端url转换为另一种页面的汽车之家pc端url
|
||||||
|
*
|
||||||
|
* @param string $url
|
||||||
|
* @return string
|
||||||
|
* @throws GuzzleException
|
||||||
|
*/
|
||||||
|
public function getChe168UrlByPc(string $url): string
|
||||||
|
{
|
||||||
|
// 解析URL获取查询参数部分
|
||||||
|
$queryString = parse_url($url, PHP_URL_QUERY);
|
||||||
|
// 将查询参数解析为数组
|
||||||
|
parse_str($queryString, $params);
|
||||||
|
|
||||||
|
// 获取infoid
|
||||||
|
$carId = $params['infoid'] ?? null;
|
||||||
|
|
||||||
|
$pcUrl = 'https://www.che168.com/dealer/111/' . $carId . '.html?pvareaid=0';
|
||||||
|
|
||||||
|
return $pcUrl;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 从移动端url获取sku_id
|
* 从移动端url获取sku_id
|
||||||
* @param string $url
|
* @param string $url
|
||||||
@@ -219,18 +241,18 @@ class UsedCarCrawlSingle
|
|||||||
$attribute[$title_key[$field]] = str_replace($field, '', $value);
|
$attribute[$title_key[$field]] = str_replace($field, '', $value);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 车辆图片
|
|
||||||
// $img = $item->find('.list-photo img')->attr('src');
|
|
||||||
|
|
||||||
return $attribute;
|
return $attribute;
|
||||||
})->all();
|
})->all();
|
||||||
|
|
||||||
$cars = array_filter($cars);
|
$cars = array_filter($cars);
|
||||||
$cars = array_merge(...$cars);
|
$cars = array_merge(...$cars);
|
||||||
$cars['product_name'] = $cars['car_name'] = $ql->find('div.car-box .car-brand-name')->text();
|
$cars['product_name'] = $cars['car_name'] = $ql->find('div.car-box .car-brand-name')->text();
|
||||||
$cars['price'] = $ql->find('span.price')->text();
|
$cars['price'] = $ql->find('div.goodstartmoney')->text();
|
||||||
$cars['price'] = explode('万', $cars['price'])[0]??0;
|
if(empty($cars['price'])){
|
||||||
$cars['price'] = str_replace(['¥', '¥'], '', $cars['price']);
|
$cars['price'] = $ql->find('span#overlayPrice')->text();
|
||||||
|
}
|
||||||
|
// $cars['price'] = explode('万', $cars['price'])[0]??0;
|
||||||
|
$cars['price'] = str_replace(['¥', '¥', '万'], '', $cars['price']);
|
||||||
$cars['price'] = $cars['price']*10000;
|
$cars['price'] = $cars['price']*10000;
|
||||||
//$cars['price'] = $ql->find('#overlayPrice')->text();
|
//$cars['price'] = $ql->find('#overlayPrice')->text();
|
||||||
$cars['title'] = $cars['product_name'];
|
$cars['title'] = $cars['product_name'];
|
||||||
@@ -254,12 +276,13 @@ class UsedCarCrawlSingle
|
|||||||
$cars['year'] = !empty($year) ? str_replace('款', '', $year):'';
|
$cars['year'] = !empty($year) ? str_replace('款', '', $year):'';
|
||||||
|
|
||||||
//图片
|
//图片
|
||||||
$images = $ql->find('#pic_li img')->map(function ($img) {
|
$images = $ql->find('.swiper-container2 .swiper-slide .jiaodianphotoclick img')->map(function ($img) {
|
||||||
// 只取 data-original
|
// 优先尝试 data-original(懒加载图片)
|
||||||
$src = $img->attr('data-original');
|
$src = $img->attr('src');
|
||||||
|
|
||||||
if (!$src) {
|
// 如果没有 data-original,尝试 src
|
||||||
return null;
|
if (!$src || $src === '' ||str_contains($src, 'default-che168')) {
|
||||||
|
$src = $img->attr('data-original');
|
||||||
}
|
}
|
||||||
|
|
||||||
// 补全 https
|
// 补全 https
|
||||||
@@ -267,9 +290,18 @@ class UsedCarCrawlSingle
|
|||||||
$src = 'https:' . $src;
|
$src = 'https:' . $src;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 过滤掉占位符图片
|
||||||
|
if (str_contains($src, 'default-che168')) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
return $src;
|
return $src;
|
||||||
})->filter()->values()->all();
|
})->filter()->values()->all();
|
||||||
|
|
||||||
|
if (empty($images)) {
|
||||||
|
Log::warning("crawlChe168Car:未找到任何车辆图片,URL: {$url}");
|
||||||
|
}
|
||||||
|
|
||||||
$saveDir = public_path('uploads/usedcar/' . $carId);
|
$saveDir = public_path('uploads/usedcar/' . $carId);
|
||||||
$cars['used_car_id'] = $carId;
|
$cars['used_car_id'] = $carId;
|
||||||
$image = $this->uploadAndRemove($images, $saveDir);
|
$image = $this->uploadAndRemove($images, $saveDir);
|
||||||
|
|||||||
Reference in New Issue
Block a user