反爬字段处理
This commit is contained in:
@@ -4,6 +4,8 @@ namespace app\common\utils\crawl\service;
|
||||
|
||||
use app\common\model\UsedcarCrawlResultsImages;
|
||||
use app\common\service\UploadService;
|
||||
use app\common\traits\crud\CrudServiceTrait;
|
||||
use app\common\utils\crawl\service\UnCodeFont;
|
||||
use GuzzleHttp\Client;
|
||||
use GuzzleHttp\Exception\GuzzleException;
|
||||
use QL\QueryList;
|
||||
@@ -13,6 +15,9 @@ use Throwable;
|
||||
|
||||
class UsedCarCrawlSingle
|
||||
{
|
||||
use CrudServiceTrait;
|
||||
use UnCodeFont;
|
||||
|
||||
// 默认请求头
|
||||
private const DEFAULT_HEADERS = [
|
||||
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
@@ -284,7 +289,6 @@ class UsedCarCrawlSingle
|
||||
$images = $this->collectChe168Images($ql, '#pic_li img', true);
|
||||
}
|
||||
$images = array_values(array_unique($images));
|
||||
var_dump($images);
|
||||
|
||||
if (empty($images)) {
|
||||
Log::warning("crawlChe168Car:未找到任何车辆图片,URL: {$url}");
|
||||
@@ -485,6 +489,9 @@ class UsedCarCrawlSingle
|
||||
|
||||
$title = trim($ql->find('h1')->text());
|
||||
$price = trim($ql->find('.head-info_price-wrap__Y4bxi > p')->text());
|
||||
if(empty($price)){
|
||||
$price = trim($ql->find('span.tw-text-color-red-500')->text());
|
||||
}
|
||||
$pageText = $ql->find('body')->text();
|
||||
$jsonText = $ql->find('#__NEXT_DATA__')->text();
|
||||
|
||||
@@ -560,7 +567,13 @@ class UsedCarCrawlSingle
|
||||
$conditionDesc = trim($m[1]);
|
||||
}
|
||||
|
||||
|
||||
$price = str_replace('一口价', '', $price);
|
||||
if ($this->needFontDecode($price)) {
|
||||
$price = $this->decodeObfuscatedText($price, []);
|
||||
}
|
||||
if ($this->needFontDecode($usageMileage)) {
|
||||
$price = $this->decodeObfuscatedText($usageMileage, []);
|
||||
}
|
||||
return [
|
||||
'used_car_id' => $this->extractIdFromUrl($url),
|
||||
'car_id' => $carId,
|
||||
|
||||
Reference in New Issue
Block a user