汽车之家抓取
This commit is contained in:
@@ -11,6 +11,7 @@ use app\common\service\UsedCarService;
|
||||
use app\common\utils\crawl\service\UsedCarCrawlSingle;
|
||||
use app\common\utils\pdf\service\MpdfDriver;
|
||||
use support\exception\BusinessException;
|
||||
use support\Log;
|
||||
use support\Request;
|
||||
|
||||
class UsedCarController extends BaseController
|
||||
@@ -186,7 +187,8 @@ class UsedCarController extends BaseController
|
||||
if (str_contains($e->getMessage(), '404 Not Found')){
|
||||
return $this->error('车辆已下架或找不到页面');
|
||||
}
|
||||
return $this->error($e->getMessage());
|
||||
Log::error('crawlCar fail',['param' => $request->get('url'),'message' => $e->getMessage().$e->getFile().$e->getLine()]);
|
||||
return $this->error($e->getMessage().$e->getFile().$e->getLine());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -447,7 +447,19 @@ class UsedCarService
|
||||
}
|
||||
$info = $usedCarCrawl->crawlCar($finalUrl);
|
||||
break;
|
||||
case str_contains($url,'che168.com/dealer'): //汽车之家
|
||||
case str_contains($url,'www.che168.com/dealer'): //汽车之家
|
||||
$info = $usedCarCrawl->crawlChe168Car($url);
|
||||
break;
|
||||
case str_contains($url,'wap.che168.com/dealer'): //汽车之家移动端链接
|
||||
$finalUrl = $usedCarCrawl->getChe168UrlByMobile($url);
|
||||
$info = $usedCarCrawl->crawlChe168Car($finalUrl);
|
||||
break;
|
||||
case str_contains($url,'comm.app.autohome.com.cn'): //汽车之家移动端链接
|
||||
$finalUrl = $usedCarCrawl->get168FinaUrl($url);
|
||||
var_dump($finalUrl);
|
||||
$info = $usedCarCrawl->crawlChe168Car($finalUrl);
|
||||
break;
|
||||
case str_contains($url,'www.che168.com/dealer'): //汽车之家
|
||||
$info = $usedCarCrawl->crawlChe168Car($url);
|
||||
break;
|
||||
default:
|
||||
|
||||
@@ -59,7 +59,62 @@ class UsedCarCrawlSingle
|
||||
$finalUrl = end($history);
|
||||
|
||||
$pcUrl = $this->getPcUrl($finalUrl);
|
||||
return $pcUrl;
|
||||
}
|
||||
|
||||
/**
|
||||
* 通过汽车之家分享链接获取pc端地址
|
||||
*
|
||||
* @param string $url
|
||||
* @return string
|
||||
* @throws GuzzleException
|
||||
*/
|
||||
public function get168FinaUrl(string $url): string
|
||||
{
|
||||
$client = new Client([
|
||||
'verify' => false,
|
||||
'timeout' => 10,
|
||||
'headers' => [
|
||||
'User-Agent' => 'Mozilla/5.0 (Linux; Android 13; Mobile)'
|
||||
]
|
||||
]);
|
||||
|
||||
$response = $client->get($url);
|
||||
$html = (string) $response->getBody();
|
||||
|
||||
preg_match(
|
||||
'/var\s+oUrl\s*=\s*[\'"]([^\'"]+che168\.com[^\'"]+)[\'"]/',
|
||||
$html,
|
||||
$m
|
||||
);
|
||||
|
||||
if(empty($m[1])){ //如果没有获取到移动端地址
|
||||
throw new \Exception('168移动端地址解析失败');
|
||||
}
|
||||
|
||||
$pcUrl = $this->getChe168UrlByMobile($m[1]);
|
||||
return $pcUrl;
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据移动端url获取汽车之家pc端url
|
||||
*
|
||||
* @param string $url
|
||||
* @return string
|
||||
* @throws GuzzleException
|
||||
*/
|
||||
public function getChe168UrlByMobile(string $url): string
|
||||
{
|
||||
|
||||
// 解析 URL获取车辆id
|
||||
$parsedUrl = parse_url($url);
|
||||
$path = $parsedUrl['path']; // "/dealer/162138/56723550.html"
|
||||
// 使用 pathinfo 获取文件名
|
||||
$pathParts = explode('/', $path);
|
||||
$filename = end($pathParts); // "56723550.html"
|
||||
$carId = pathinfo($filename, PATHINFO_FILENAME); // "56723550"
|
||||
|
||||
$pcUrl = 'https://www.che168.com/dealer/111/' . $carId . '.html?pvareaid=0';
|
||||
return $pcUrl;
|
||||
}
|
||||
|
||||
@@ -143,6 +198,7 @@ class UsedCarCrawlSingle
|
||||
// 请求 HTML
|
||||
$response = $client->get($url);
|
||||
$html = (string)$response->getBody();
|
||||
file_put_contents(runtime_path('craw.html'), $html);
|
||||
|
||||
// 检查是否获取到 HTML
|
||||
if (empty($html)) {
|
||||
@@ -151,7 +207,6 @@ class UsedCarCrawlSingle
|
||||
|
||||
// 使用 QueryList 解析 HTML
|
||||
$ql = QueryList::html($html);
|
||||
file_put_contents(runtime_path('168.html'),$html);
|
||||
|
||||
// 解析车辆列表，每辆车可能在 li.cards-li 或类似类名里
|
||||
$cars = $ql->find('div#nav1 ul.basic-item-ul li')->map(function($item) {
|
||||
@@ -174,7 +229,9 @@ class UsedCarCrawlSingle
|
||||
$cars = array_merge(...$cars);
|
||||
$cars['product_name'] = $cars['car_name'] = $ql->find('div.car-box .car-brand-name')->text();
|
||||
$cars['price'] = $ql->find('span.price')->text();
|
||||
$cars['price'] = !empty($cars['price'])?str_replace('万', '', $cars['price'])*10000:0;
|
||||
$cars['price'] = explode('万', $cars['price'])[0]??0;
|
||||
$cars['price'] = str_replace(['¥', '￥'], '', $cars['price']);
|
||||
$cars['price'] = $cars['price']*10000;
|
||||
//$cars['price'] = $ql->find('#overlayPrice')->text();
|
||||
$cars['title'] = $cars['product_name'];
|
||||
$cars['condition_desc'] = $ql->find('div.leave-message-box #messageBox')->text();
|
||||
@@ -184,9 +241,9 @@ class UsedCarCrawlSingle
|
||||
$links = $ql->find('.bread-crumbs a')->map(function ($a) {
|
||||
return trim($a->text());
|
||||
})->all();
|
||||
$cars['brand_name'] = $links[3] ?str_replace('二手','',$links[3]): null;
|
||||
$cars['brand_name'] = !empty($links[3]) ? str_replace('二手','',$links[3]): null;
|
||||
$cars['brand_id'] = 0;
|
||||
$cars['series_name'] = $links[4] ?str_replace('二手','',$links[4]): null;
|
||||
$cars['series_name'] = !empty($links[4]) ?str_replace('二手','',$links[4]): null;
|
||||
$cars['series_id'] = 0;
|
||||
$cars['car_id'] = 0;
|
||||
$info = explode(' ',$cars['product_name']);
|
||||
|
||||
Reference in New Issue
Block a user