alibaba

2025-12-22 05:57:11 +00:00
parent 56eabe4e32
commit 149882b525
9 changed files with 623 additions and 0 deletions
@@ -0,0 +1,82 @@
+<?php
+namespace app\alibaba\controller;
+
+use support\Request;
+use support\Response;
+use app\alibaba\service\InquiryScraperService;
+
+class InquiryScraperController
+{
+    private InquiryScraperService $svc;
+
+    public function __construct()
+    {
+        $this->svc = new InquiryScraperService();
+    }
+
+    /** 列表上报 */
+    public function reportList(Request $request): Response
+    {
+        $payload = $request->post();
+        $ret = $this->svc->handleListReport($payload);
+        return json([
+            'code' => $ret['ok'] ? 200 : 400,
+            'msg'  => $ret['ok'] ? 'ok' : ($ret['err'] ?? 'error'),
+            'data' => $ret,
+        ]);
+    }
+
+    /** 详情上报（支持分块 chunk） */
+    public function reportDetail(Request $request): Response
+    {
+        $payload = $request->post();
+
+        var_dump($payload);
+
+        $ret = $this->svc->handleDetailReport($payload);
+        return json([
+            'code' => $ret['ok'] ? 200 : 400,
+            'msg'  => $ret['ok'] ? 'ok' : ($ret['err'] ?? 'error'),
+            'data' => $ret,
+        ]);
+    }
+
+    /**
+     * 媒体上传：multipart/form-data
+     * fields:
+     *  taskId, detailUrl, msgId, mediaType, sourceUrl, md5, size, fileName
+     * file:
+     *  file
+     */
+    public function uploadMedia(Request $request): Response
+    {
+        $ret = $this->svc->handleUploadMedia($request);
+        return json([
+            'code' => $ret['ok'] ? 200 : 400,
+            'msg'  => $ret['ok'] ? 'ok' : ($ret['err'] ?? 'error'),
+            'data' => $ret,
+        ]);
+    }
+
+    /** 可选：本地媒体访问 */
+    public function serveMedia(Request $request, string $path): Response
+    {
+        return $this->svc->serveMediaFile($path);
+    }
+
+
+
+    public function reportContact(Request $request)
+    {
+
+        var_dump($request->post());
+
+        return json([
+            'code' => 200,
+            'msg'  => 'ok',
+            'data' => [
+                'ok' => true,
+            ],
+        ]);
+    }
+}
@@ -0,0 +1,14 @@
+<?php
+namespace app\alibaba\model;
+
+use Illuminate\Database\Eloquent\Model;
+
+class InquiryDetailMedia extends Model
+{
+    protected $table = 'inquiry_detail_media';
+    public $timestamps = false;
+
+    protected $fillable = [
+        'session_id','message_id','media_type','source_url','file_name','file_ext','size','md5','storage_path','public_url','created_at','updated_at'
+    ];
+}
@@ -0,0 +1,19 @@
+<?php
+namespace app\alibaba\model;
+
+use Illuminate\Database\Eloquent\Model;
+
+class InquiryDetailMessage extends Model
+{
+    protected $table = 'inquiry_detail_message';
+    public $timestamps = false;
+
+    protected $fillable = [
+        'session_id','msg_id','direction','time_text','original_raw','text_shown','translated',
+        'is_card','hrefs_json','read_status','chunk_index','created_at','updated_at'
+    ];
+
+    protected $casts = [
+        'hrefs_json' => 'array',
+    ];
+}
@@ -0,0 +1,19 @@
+<?php
+namespace app\alibaba\model;
+
+use Illuminate\Database\Eloquent\Model;
+
+class InquiryDetailSession extends Model
+{
+    protected $table = 'inquiry_detail_session';
+    public $timestamps = false;
+
+    protected $fillable = [
+        'task_id','list_item_id','detail_url','inquiry_id','customer_json','messages_count','status','raw_json','created_at','updated_at'
+    ];
+
+    protected $casts = [
+        'customer_json' => 'array',
+        'raw_json' => 'array',
+    ];
+}
@@ -0,0 +1,18 @@
+<?php
+namespace app\alibaba\model;
+
+use Illuminate\Database\Eloquent\Model;
+
+class InquiryListItem extends Model
+{
+    protected $table = 'inquiry_list_item';
+    public $timestamps = false;
+
+    protected $fillable = [
+        'task_id','page_id','page','idx','inquiry_id','detail_url','buyer_name','country','subject','snippet','raw_json','created_at','updated_at'
+    ];
+
+    protected $casts = [
+        'raw_json' => 'array',
+    ];
+}
@@ -0,0 +1,18 @@
+<?php
+namespace app\alibaba\model;
+
+use Illuminate\Database\Eloquent\Model;
+
+class InquiryListPage extends Model
+{
+    protected $table = 'inquiry_list_page';
+    public $timestamps = false;
+
+    protected $fillable = [
+        'task_id','page','total_on_page','is_done','raw_json','created_at','updated_at'
+    ];
+
+    protected $casts = [
+        'raw_json' => 'array',
+    ];
+}
@@ -0,0 +1,18 @@
+<?php
+namespace app\alibaba\model;
+
+use Illuminate\Database\Eloquent\Model;
+
+class InquiryTask extends Model
+{
+    protected $table = 'inquiry_task';
+    public $timestamps = false;
+
+    protected $fillable = [
+        'task_id','mode','start_page','max_pages','status','server_url','meta_json','created_at','updated_at'
+    ];
+
+    protected $casts = [
+        'meta_json' => 'array',
+    ];
+}
@@ -0,0 +1,422 @@
+<?php
+namespace app\alibaba\service;
+
+use support\Db;
+use support\Request;
+use support\Response;
+use app\alibaba\model\InquiryTask;
+use app\alibaba\model\InquiryListPage;
+use app\alibaba\model\InquiryListItem;
+use app\alibaba\model\InquiryDetailSession;
+use app\alibaba\model\InquiryDetailMessage;
+use app\alibaba\model\InquiryDetailMedia;
+
+class InquiryScraperService
+{
+    /** 列表上报：写 task + list_page + list_item（幂等） */
+    public function handleListReport(array $p): array
+    {
+        $now = time();
+        $taskId = (string)($p['taskId'] ?? '');
+        if ($taskId === '') return ['ok' => false, 'err' => 'taskId required'];
+
+        $page = (int)($p['page'] ?? 1);
+        $serverUrl = (string)($p['serverUrl'] ?? '');
+
+        // 兼容：payload.data.list 或 payload.list
+        $list = [];
+        if (isset($p['data']['list']) && is_array($p['data']['list'])) $list = $p['data']['list'];
+        if (!$list && isset($p['list']) && is_array($p['list'])) $list = $p['list'];
+
+        $totalOnPage = (int)($p['totalOnPage'] ?? count($list));
+
+        return Db::transaction(function () use ($taskId, $page, $serverUrl, $list, $totalOnPage, $p, $now) {
+            // 1) task
+            $task = InquiryTask::query()->firstOrCreate(
+                ['task_id' => $taskId],
+                [
+                    'mode' => 'list',
+                    'start_page' => $page,
+                    'max_pages' => 0,
+                    'status' => 0,
+                    'server_url' => $serverUrl,
+                    'meta_json' => null,
+                    'created_at' => $now,
+                    'updated_at' => $now,
+                ]
+            );
+            $task->mode = 'list';
+            if ($serverUrl) $task->server_url = $serverUrl;
+            $task->updated_at = $now;
+            $task->save();
+
+            // 2) list_page（唯一：task_id+page）
+            $pageRow = InquiryListPage::query()->updateOrCreate(
+                ['task_id' => $taskId, 'page' => $page],
+                [
+                    'total_on_page' => $totalOnPage,
+                    'is_done' => 1,
+                    'raw_json' => $this->trimListPageRaw($p),
+                    'created_at' => $now,
+                    'updated_at' => $now,
+                ]
+            );
+
+            // 3) list_item（唯一：page_id+idx）
+            $n = 0;
+            foreach ($list as $i => $row) {
+                if (!is_array($row)) continue;
+
+                $idx = (int)($row['idx'] ?? $i);
+                $detailUrl = (string)($row['detailUrl'] ?? $row['url'] ?? '');
+                // $inquiryId = (string)($row['inquiryId'] ?? $row['imInquiryId'] ?? '');
+                // ✅ 只信 imInquiryId，避免把 encrypt/hash 长串塞进 inquiry_id
+                $inquiryId = (string)($row['imInquiryId'] ?? '');
+                if (strlen($inquiryId) > 255) $inquiryId = ''; // 兜底
+
+
+                InquiryListItem::query()->updateOrCreate(
+                    ['page_id' => $pageRow->id, 'idx' => $idx],
+                    [
+                        'task_id' => $taskId,
+                        'page' => $page,
+                        'inquiry_id' => $inquiryId,
+                        'detail_url' => $detailUrl,
+                        'buyer_name' => (string)($row['buyerName'] ?? ''),
+                        'country' => (string)($row['country'] ?? ''),
+                        'subject' => (string)($row['subject'] ?? ''),
+                        'snippet' => (string)($row['snippet'] ?? ''),
+                        'raw_json' => $row,
+                        'created_at' => $now,
+                        'updated_at' => $now,
+                    ]
+                );
+                $n++;
+            }
+
+            return ['ok' => true, 'page' => $page, 'items' => $n, 'listPageId' => (int)$pageRow->id];
+        });
+    }
+
+    /** 详情上报：写 task + detail_session + detail_message（分块幂等）+（可选）media 元信息绑定 */
+    public function handleDetailReport(array $p): array
+    {
+        $now = time();
+        $taskId = (string)($p['taskId'] ?? '');
+        if ($taskId === '') return ['ok' => false, 'err' => 'taskId required'];
+
+        $serverUrl = (string)($p['serverUrl'] ?? '');
+
+        $detail = $p['detail'] ?? null;
+        if (!is_array($detail)) return ['ok' => false, 'err' => 'detail required'];
+
+        $detailUrl = (string)($detail['url'] ?? '');
+        if ($detailUrl === '') return ['ok' => false, 'err' => 'detail.url required'];
+
+        // $inquiryId = (string)($detail['inquiryId'] ?? $this->extractImInquiryId($detailUrl));
+        // ✅ 只从 detailUrl 取 imInquiryId，取不到就空（不要 fallback 到其它字段）
+        $inquiryId = (string)($this->extractImInquiryId($detailUrl) ?: '');
+        if (strlen($inquiryId) > 255) $inquiryId = ''; // 兜底
+
+
+        $customer  = $detail['customer'] ?? null;
+
+        $messages = $detail['messages'] ?? [];
+        if (!is_array($messages)) $messages = [];
+
+        // chunk info
+        $chunk = $detail['chunk'] ?? [];
+        $chunkIndex = (int)($chunk['index'] ?? $chunk['seq'] ?? 0);
+        $isFinal = !empty($chunk['isFinal']);
+        $messagesCount = (int)($detail['messagesCount'] ?? 0);
+
+        // （可选）如果插件传 listItemId 或者 listRow 的定位信息
+        $listItemId = (int)($detail['listItemId'] ?? $p['listItemId'] ?? 0);
+
+        return Db::transaction(function () use (
+            $taskId, $serverUrl, $detailUrl, $inquiryId, $customer,
+            $messages, $chunkIndex, $isFinal, $messagesCount, $listItemId, $detail, $now
+        ) {
+            // 1) task
+            $task = InquiryTask::query()->firstOrCreate(
+                ['task_id' => $taskId],
+                [
+                    'mode' => 'detail',
+                    'start_page' => 1,
+                    'max_pages' => 0,
+                    'status' => 0,
+                    'server_url' => $serverUrl,
+                    'meta_json' => null,
+                    'created_at' => $now,
+                    'updated_at' => $now,
+                ]
+            );
+            $task->mode = 'detail';
+            if ($serverUrl) $task->server_url = $serverUrl;
+            $task->updated_at = $now;
+            $task->save();
+
+            // 2) session（唯一：task_id+detail_url）
+            $session = InquiryDetailSession::query()->firstOrCreate(
+                ['task_id' => $taskId, 'detail_url' => $detailUrl],
+                [
+                    'list_item_id' => $listItemId,
+                    'inquiry_id' => $inquiryId,
+                    'customer_json' => $customer,
+                    'messages_count' => 0,
+                    'status' => 0,
+                    'raw_json' => null,
+                    'created_at' => $now,
+                    'updated_at' => $now,
+                ]
+            );
+
+            // 更新 session 头信息
+            if ($listItemId && (int)$session->list_item_id === 0) $session->list_item_id = $listItemId;
+            if ($inquiryId) $session->inquiry_id = $inquiryId;
+            $session->customer_json = $customer;
+            $session->raw_json = $this->trimDetailRaw($detail);
+            $session->updated_at = $now;
+            $session->save();
+
+            // 3) messages upsert（唯一：session_id+msg_id）
+            $upserted = 0;
+            $mediaBound = 0;
+
+            foreach ($messages as $m) {
+                if (!is_array($m)) continue;
+                $msgId = (string)($m['msgId'] ?? '');
+                if ($msgId === '') continue;
+
+                $msgRow = InquiryDetailMessage::query()->updateOrCreate(
+                    ['session_id' => $session->id, 'msg_id' => $msgId],
+                    [
+                        'direction' => (string)($m['direction'] ?? ''),
+                        'time_text' => (string)($m['timeText'] ?? ''),
+                        'original_raw' => (string)($m['originalRaw'] ?? ''),
+                        'text_shown' => (string)($m['textShown'] ?? ''),
+                        'translated' => (string)($m['translated'] ?? ''),
+                        'is_card' => (int)(!empty($m['isCard']) ? 1 : 0),
+                        'hrefs_json' => $m['hrefs'] ?? [],
+                        'read_status' => (string)($m['readStatus'] ?? ''),
+                        'chunk_index' => $chunkIndex,
+                        'created_at' => $now,
+                        'updated_at' => $now,
+                    ]
+                );
+                $upserted++;
+
+                // 如果插件在 message 上携带 medias 元信息：先绑定 media 行（文件真正上传由 /media/upload）
+                if (!empty($m['medias']) && is_array($m['medias'])) {
+                    foreach ($m['medias'] as $media) {
+                        if (!is_array($media)) continue;
+                        $sourceUrl = (string)($media['url'] ?? $media['sourceUrl'] ?? '');
+                        if ($sourceUrl === '') continue;
+
+                        InquiryDetailMedia::query()->updateOrCreate(
+                            [
+                                'session_id' => $session->id,
+                                'message_id' => $msgRow->id,
+                                'source_url' => $sourceUrl,
+                            ],
+                            [
+                                'media_type' => (string)($media['type'] ?? 'file'),
+                                'file_name' => (string)($media['name'] ?? ''),
+                                'file_ext' => (string)($media['ext'] ?? ''),
+                                'size' => (int)($media['size'] ?? 0),
+                                'md5' => (string)($media['md5'] ?? ''),
+                                'storage_path' => (string)($media['storagePath'] ?? ''),
+                                'public_url' => (string)($media['publicUrl'] ?? ''),
+                                'created_at' => $now,
+                                'updated_at' => $now,
+                            ]
+                        );
+                        $mediaBound++;
+                    }
+                }
+            }
+
+            // 4) final：统计 messages_count + 标记 session 完成
+            if ($isFinal) {
+                $cnt = InquiryDetailMessage::query()->where('session_id', $session->id)->count();
+                $session->messages_count = max($messagesCount, (int)$cnt);
+                $session->status = 2; // 完成
+                $session->updated_at = $now;
+                $session->save();
+            }
+
+            return [
+                'ok' => true,
+                'sessionId' => (int)$session->id,
+                'chunk' => ['index' => $chunkIndex, 'isFinal' => $isFinal],
+                'messagesUpserted' => $upserted,
+                'mediasBound' => $mediaBound,
+            ];
+        });
+    }
+
+    /** 媒体上传：multipart，保存文件并回填 inquiry_detail_media.storage_path/public_url */
+    public function handleUploadMedia(Request $request): array
+    {
+        $now = time();
+
+        $taskId    = (string)$request->post('taskId', '');
+        $detailUrl = (string)$request->post('detailUrl', '');
+        $msgId     = (string)$request->post('msgId', '');
+        $mediaType = (string)$request->post('mediaType', 'file');
+        $sourceUrl = (string)$request->post('sourceUrl', '');
+        $md5       = (string)$request->post('md5', '');
+        $size      = (int)$request->post('size', 0);
+        $fileName  = (string)$request->post('fileName', '');
+
+        if ($taskId === '' || $detailUrl === '' || $msgId === '') {
+            return ['ok' => false, 'err' => 'taskId/detailUrl/msgId required'];
+        }
+
+        $file = $request->file('file');
+        if (!$file) return ['ok' => false, 'err' => 'file missing'];
+
+        // 保存目录：runtime/uploads/inquiry_media/YYYYMM/
+        $ym = date('Ym');
+        $dir = base_path() . "/runtime/uploads/inquiry_media/{$ym}";
+        if (!is_dir($dir)) @mkdir($dir, 0777, true);
+
+        $originalName = $fileName ?: ($file->getUploadName() ?: ('media_' . $now));
+        $ext = strtolower(pathinfo($originalName, PATHINFO_EXTENSION));
+        $safeExt = $ext ? preg_replace('/[^a-z0-9]/i', '', $ext) : '';
+
+        $saveName = 't' . $taskId . '_' . md5($originalName . $now . random_int(1, 999999)) . ($safeExt ? ".{$safeExt}" : '');
+        $file->move($dir . '/' . $saveName);
+
+        // 注意：serveMedia 是从 runtime/uploads/ 下面读，所以这里用相对 uploads 的路径
+        // full: runtime/uploads/inquiry_media/YYYYMM/xxx.ext
+        $storageRel = "inquiry_media/{$ym}/{$saveName}";
+        $storagePath = "/runtime/uploads/{$storageRel}";
+        $publicUrl = "/media/{$storageRel}";
+
+        return Db::transaction(function () use (
+            $taskId, $detailUrl, $msgId, $mediaType, $sourceUrl, $md5, $size, $originalName, $safeExt,
+            $storagePath, $publicUrl, $now
+        ) {
+            // session：确保存在
+            $session = InquiryDetailSession::query()->firstOrCreate(
+                ['task_id' => $taskId, 'detail_url' => $detailUrl],
+                [
+                    'list_item_id' => 0,
+                    'inquiry_id' => $this->extractImInquiryId($detailUrl),
+                    'customer_json' => null,
+                    'messages_count' => 0,
+                    'status' => 0,
+                    'raw_json' => null,
+                    'created_at' => $now,
+                    'updated_at' => $now,
+                ]
+            );
+
+            // message：确保存在（某些媒体消息可能先传媒体再传 message）
+            $msgRow = InquiryDetailMessage::query()->firstOrCreate(
+                ['session_id' => $session->id, 'msg_id' => $msgId],
+                [
+                    'direction' => '',
+                    'time_text' => '',
+                    'original_raw' => '',
+                    'text_shown' => '',
+                    'translated' => '',
+                    'is_card' => 1,
+                    'hrefs_json' => [],
+                    'read_status' => '',
+                    'chunk_index' => 0,
+                    'created_at' => $now,
+                    'updated_at' => $now,
+                ]
+            );
+
+            // media：幂等（唯一：session_id+message_id+source_url）
+            $uniqSource = $sourceUrl ?: $storagePath;
+
+            $media = InquiryDetailMedia::query()->updateOrCreate(
+                ['session_id' => $session->id, 'message_id' => $msgRow->id, 'source_url' => $uniqSource],
+                [
+                    'media_type' => $mediaType,
+                    'source_url' => $uniqSource,
+                    'file_name' => $originalName,
+                    'file_ext' => $safeExt,
+                    'size' => $size,
+                    'md5' => $md5,
+                    'storage_path' => $storagePath,
+                    'public_url' => $publicUrl,
+                    'created_at' => $now,
+                    'updated_at' => $now,
+                ]
+            );
+
+            return [
+                'ok' => true,
+                'mediaId' => (int)$media->id,
+                'storage_path' => $storagePath,
+                'public_url' => $publicUrl,
+                'file_name' => $originalName,
+            ];
+        });
+    }
+
+    /** 本地媒体访问：GET /media/{path} -> runtime/uploads/{path} */
+    public function serveMediaFile(string $path): Response
+    {
+        // 防路径穿越
+        $path = str_replace(['..', '\\'], ['', '/'], $path);
+        $full = base_path() . '/runtime/uploads/' . $path;
+        if (!is_file($full)) {
+            return response('Not Found', 404);
+        }
+
+        $mime = $this->guessMime($full);
+        return response(file_get_contents($full), 200, ['Content-Type' => $mime]);
+    }
+
+    private function guessMime(string $file): string
+    {
+        $ext = strtolower(pathinfo($file, PATHINFO_EXTENSION));
+        return match ($ext) {
+            'jpg','jpeg' => 'image/jpeg',
+            'png' => 'image/png',
+            'gif' => 'image/gif',
+            'webp' => 'image/webp',
+            'mp4' => 'video/mp4',
+            'pdf' => 'application/pdf',
+            'xls' => 'application/vnd.ms-excel',
+            'xlsx' => 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+            'doc' => 'application/msword',
+            'docx' => 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+            'ppt' => 'application/vnd.ms-powerpoint',
+            'pptx' => 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+            'zip' => 'application/zip',
+            default => 'application/octet-stream'
+        };
+    }
+
+    private function extractImInquiryId(string $url): string
+    {
+        if ($url === '') return '';
+        $m = [];
+        if (preg_match('/[?&]imInquiryId=([^&]+)/i', $url, $m)) return (string)$m[1];
+        return '';
+    }
+
+    /** 裁剪 list_page raw_json（避免存超大） */
+    private function trimListPageRaw(array $p): array
+    {
+        $keep = $p;
+        if (isset($keep['list'])) unset($keep['list']);
+        if (isset($keep['data']['list'])) unset($keep['data']['list']);
+        return $keep;
+    }
+
+    /** 裁剪 detail raw_json（避免存超大 messages） */
+    private function trimDetailRaw(array $detail): array
+    {
+        $keep = $detail;
+        if (isset($keep['messages'])) unset($keep['messages']);
+        return $keep;
+    }
+}
@@ -19,3 +19,16 @@ use Webman\Route;
 Route::any('/ts/form', [app\controller\TransmitController::class, 'form']);

 Route::any('/iq/form', [app\inquery\controller\InqueryController::class, 'form']);
+
+
+
+Route::any('/inquiry/list/report', [app\alibaba\controller\InquiryScraperController::class, 'reportList']);
+Route::any('/inquiry/detail/report', [app\alibaba\controller\InquiryScraperController::class, 'reportDetail']);
+Route::any('/inquiry/detail/media/upload', [app\alibaba\controller\InquiryScraperController::class, 'uploadMedia']);
+
+// 可选：本地媒体访问（插件上传后可直接访问）
+Route::any('/media/{path:.+}', [app\alibaba\controller\InquiryScraperController::class, 'serveMedia']);
+
+
+
+Route::any('/inquiry/contact/report', [app\alibaba\controller\InquiryScraperController::class, 'reportContact']);