xinyouhe/useds
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

反爬字段处理

Browse Source
This commit is contained in:
WIN-C27C5MSVTPO\16961
2026-02-05 19:25:56 +08:00
parent a469cb810d
commit 6a9c4b1726
5 changed files with 117 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/common/controller/UsedCarController.php
+6 -4
View File
@@ -8,7 +8,8 @@ use app\common\service\OssService;
use app\common\service\SparkService;
use app\common\traits\crud\CrudControllerTrait;
use app\common\service\UsedCarService;
use app\common\utils\crawl\service\UsedCarCrawlSingle;
use app\common\utils\crawl\service\service\UsedCarCrawlSingle;
use app\common\utils\crawl\service\UnCodeFont;
use app\common\utils\pdf\service\MpdfDriver;
use support\exception\BusinessException;
use support\Log;
@@ -17,6 +18,7 @@ use support\Request;
class UsedCarController extends BaseController
{
use CrudControllerTrait;
use UnCodeFont;
protected $service;
@@ -160,10 +162,10 @@ class UsedCarController extends BaseController
$data = [
'user_id' => $request->user['id'],
'product_name' => $info['title'],
'price' => $info['price']<200 ? $info['price']*10000 : $info['price'],
'price' => $info['price']<200 ? ((float)$info['price'])*10000 : $info['price'],
'source_place' => $info['source_place'],
'displacement' => $info['capacity']??'', //排量
'mileage' => $info['usage_mileage']*10000, //里程
'mileage' => ((float)$info['usage_mileage'])*10000, //里程
'power' => $info['horsepower']??'', //马力
'body_color' => $info['body_color'],
'images' => $info['image'],
@@ -188,7 +190,7 @@ class UsedCarController extends BaseController
return $this->error('车辆已下架或找不到页面');
}
Log::error('crawlCar fail',['param' => $request->get('url'),'message' => $e->getMessage().$e->getFile().$e->getLine()]);
return $this->error($e->getMessage());
return $this->error($e->getMessage().$e->getFile().$e->getLine());
}
}
app/common/utils/crawl/service/UnCodeFont.php
+94
View File
@@ -0,0 +1,94 @@
<?php
namespace app\common\utils\crawl\service;
use support\Db;
trait UnCodeFont{
/**
* 是否包含 PUA 字符（U+E000~U+F8FF）
*/
function needFontDecode(string $text): bool
{
if ($text === '') return false;
$chars = preg_split('//u', $text, -1, PREG_SPLIT_NO_EMPTY);
if (!$chars) return false;
foreach ($chars as $ch) {
$u = mb_convert_encoding($ch, 'UCS-4BE', 'UTF-8');
if ($u === false || strlen($u) !== 4) continue;
$cp = unpack('N', $u)[1];
if ($cp >= 0xE000 && $cp <= 0xF8FF) {
return true;
}
}
return false;
}
/**
* 读取某个 font_hash 的映射表，返回 [ 'e53d' => '中', ... ]
* code 表里存的是 '&#xE53D;' 或 '&#xe53d;'
*/
function loadFontMap(string $fontHash): array
{
$rows = Db::table('font_map')
->select('code', 'char')
->where('font_hash', $fontHash)
->get()
->toArray();
$rows = array_map(function($item) {
return (array) $item;
}, $rows);
$map = [];
foreach ($rows as $r) {
// 从 '&#xE53D;' 提取 e53d
if (preg_match('/&#x([0-9a-fA-F]+);/i', $r['code'], $m)) {
$map[strtolower($m[1])] = $r['char'];
}
}
return $map;
}
/**
* 将包含 PUA 字符的字符串，按 font_map 替换
*/
function decodeObfuscatedText(string $text, array $fontMap): string
{
$fontHash = '77fd388510a974babdf4837b26d2d33f'; // 你当前字体 md5（font_map.sql 里那一份）
$fontMap = $this->loadFontMap($fontHash);
// 按 Unicode 字符切分
$chars = preg_split('//u', $text, -1, PREG_SPLIT_NO_EMPTY);
if (!$chars) return $text;
$out = '';
foreach ($chars as $ch) {
// 获取码点（十六进制），如 e53d
$hex = $this->unicodeCodePointHex($ch);
if ($hex !== null) {
$key = strtolower($hex);
// 只替换命中的（一般就是 E***）
if (isset($fontMap[$key])) {
$out .= $fontMap[$key];
continue;
}
}
// 没命中：原样保留（比如 '.' 这种正常字符）
$out .= $ch;
}
return $out;
}
/**
* 获取单个 Unicode 字符的码点十六进制（不带 0x），失败返回 null
* 例：'' -> 'e53d'
*/
function unicodeCodePointHex(string $ch): ?string
{
if ($ch === '') return null;
$u = mb_convert_encoding($ch, 'UCS-4BE', 'UTF-8');
if ($u === false || strlen($u) !== 4) return null;
$code = unpack('N', $u)[1]; // 32-bit
return dechex($code);
}
}
app/common/utils/crawl/service/UsedCarCrawl.php
+1 -1
View File
@@ -1,6 +1,6 @@
<?php
namespace app\common\utils\crawl\service;
namespace app\common\utils\crawl\service\service;
use app\common\model\UsedcarCrawlResultsImages;
use app\common\service\UploadService;
app/common/utils/crawl/service/UsedCarCrawlSingle.php
+15 -2
View File
@@ -4,6 +4,8 @@ namespace app\common\utils\crawl\service;
use app\common\model\UsedcarCrawlResultsImages;
use app\common\service\UploadService;
use app\common\traits\crud\CrudServiceTrait;
use app\common\utils\crawl\service\UnCodeFont;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException;
use QL\QueryList;
@@ -13,6 +15,9 @@ use Throwable;
class UsedCarCrawlSingle
{
use CrudServiceTrait;
use UnCodeFont;
// 默认请求头
private const DEFAULT_HEADERS = [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
@@ -284,7 +289,6 @@ class UsedCarCrawlSingle
$images = $this->collectChe168Images($ql, '#pic_li img', true);
}
$images = array_values(array_unique($images));
var_dump($images);
if (empty($images)) {
Log::warning("crawlChe168Car:未找到任何车辆图片，URL: {$url}");
@@ -485,6 +489,9 @@ class UsedCarCrawlSingle
$title = trim($ql->find('h1')->text());
$price = trim($ql->find('.head-info_price-wrap__Y4bxi > p')->text());
if(empty($price)){
$price = trim($ql->find('span.tw-text-color-red-500')->text());
}
$pageText = $ql->find('body')->text();
$jsonText = $ql->find('#__NEXT_DATA__')->text();
@@ -560,7 +567,13 @@ class UsedCarCrawlSingle
$conditionDesc = trim($m[1]);
}
$price = str_replace('一口价', '', $price);
if ($this->needFontDecode($price)) {
$price = $this->decodeObfuscatedText($price, []);
}
if ($this->needFontDecode($usageMileage)) {
$price = $this->decodeObfuscatedText($usageMileage, []);
}
return [
'used_car_id' => $this->extractIdFromUrl($url),
'car_id' => $carId,
app/controller/IndexController.php
+1 -1
View File
@@ -159,7 +159,7 @@ class IndexController
public function test(Request $request)
{
$service = new \app\common\utils\crawl\service\UsedCarCrawl();
$service = new \app\common\utils\crawl\service\service\UsedCarCrawl();
$result = $service->pushUrl('https://www.dongchedi.com/usedcar/21631642');