From 149882b525aeab18ff4912b1e7f21ff7f80e9e3a Mon Sep 17 00:00:00 2001 From: mzl Date: Mon, 22 Dec 2025 05:57:11 +0000 Subject: [PATCH] alibaba --- .../controller/InquiryScraperController.php | 82 ++++ app/alibaba/model/InquiryDetailMedia.php | 14 + app/alibaba/model/InquiryDetailMessage.php | 19 + app/alibaba/model/InquiryDetailSession.php | 19 + app/alibaba/model/InquiryListItem.php | 18 + app/alibaba/model/InquiryListPage.php | 18 + app/alibaba/model/InquiryTask.php | 18 + app/alibaba/service/InquiryScraperService.php | 422 ++++++++++++++++++ config/route.php | 13 + 9 files changed, 623 insertions(+) create mode 100644 app/alibaba/controller/InquiryScraperController.php create mode 100644 app/alibaba/model/InquiryDetailMedia.php create mode 100644 app/alibaba/model/InquiryDetailMessage.php create mode 100644 app/alibaba/model/InquiryDetailSession.php create mode 100644 app/alibaba/model/InquiryListItem.php create mode 100644 app/alibaba/model/InquiryListPage.php create mode 100644 app/alibaba/model/InquiryTask.php create mode 100644 app/alibaba/service/InquiryScraperService.php diff --git a/app/alibaba/controller/InquiryScraperController.php b/app/alibaba/controller/InquiryScraperController.php new file mode 100644 index 0000000..b41213b --- /dev/null +++ b/app/alibaba/controller/InquiryScraperController.php @@ -0,0 +1,82 @@ +svc = new InquiryScraperService(); + } + + /** 列表上报 */ + public function reportList(Request $request): Response + { + $payload = $request->post(); + $ret = $this->svc->handleListReport($payload); + return json([ + 'code' => $ret['ok'] ? 200 : 400, + 'msg' => $ret['ok'] ? 'ok' : ($ret['err'] ?? 'error'), + 'data' => $ret, + ]); + } + + /** 详情上报（支持分块 chunk） */ + public function reportDetail(Request $request): Response + { + $payload = $request->post(); + + var_dump($payload); + + $ret = $this->svc->handleDetailReport($payload); + return json([ + 'code' => $ret['ok'] ? 200 : 400, + 'msg' => $ret['ok'] ? 'ok' : ($ret['err'] ?? 'error'), + 'data' => $ret, + ]); + } + + /** + * 媒体上传：multipart/form-data + * fields: + * taskId, detailUrl, msgId, mediaType, sourceUrl, md5, size, fileName + * file: + * file + */ + public function uploadMedia(Request $request): Response + { + $ret = $this->svc->handleUploadMedia($request); + return json([ + 'code' => $ret['ok'] ? 200 : 400, + 'msg' => $ret['ok'] ? 'ok' : ($ret['err'] ?? 'error'), + 'data' => $ret, + ]); + } + + /** 可选：本地媒体访问 */ + public function serveMedia(Request $request, string $path): Response + { + return $this->svc->serveMediaFile($path); + } + + + + public function reportContact(Request $request) + { + + var_dump($request->post()); + + return json([ + 'code' => 200, + 'msg' => 'ok', + 'data' => [ + 'ok' => true, + ], + ]); + } +} \ No newline at end of file diff --git a/app/alibaba/model/InquiryDetailMedia.php b/app/alibaba/model/InquiryDetailMedia.php new file mode 100644 index 0000000..1b94fef --- /dev/null +++ b/app/alibaba/model/InquiryDetailMedia.php @@ -0,0 +1,14 @@ + 'array', + ]; +} diff --git a/app/alibaba/model/InquiryDetailSession.php b/app/alibaba/model/InquiryDetailSession.php new file mode 100644 index 0000000..b9c486e --- /dev/null +++ b/app/alibaba/model/InquiryDetailSession.php @@ -0,0 +1,19 @@ + 'array', + 'raw_json' => 'array', + ]; +} diff --git a/app/alibaba/model/InquiryListItem.php b/app/alibaba/model/InquiryListItem.php new file mode 100644 index 0000000..0959c13 --- /dev/null +++ b/app/alibaba/model/InquiryListItem.php @@ -0,0 +1,18 @@ + 'array', + ]; +} diff --git a/app/alibaba/model/InquiryListPage.php b/app/alibaba/model/InquiryListPage.php new file mode 100644 index 0000000..6cd1548 --- /dev/null +++ b/app/alibaba/model/InquiryListPage.php @@ -0,0 +1,18 @@ + 'array', + ]; +} diff --git a/app/alibaba/model/InquiryTask.php b/app/alibaba/model/InquiryTask.php new file mode 100644 index 0000000..fd4ccb5 --- /dev/null +++ b/app/alibaba/model/InquiryTask.php @@ -0,0 +1,18 @@ + 'array', + ]; +} diff --git a/app/alibaba/service/InquiryScraperService.php b/app/alibaba/service/InquiryScraperService.php new file mode 100644 index 0000000..109a289 --- /dev/null +++ b/app/alibaba/service/InquiryScraperService.php @@ -0,0 +1,422 @@ + false, 'err' => 'taskId required']; + + $page = (int)($p['page'] ?? 1); + $serverUrl = (string)($p['serverUrl'] ?? ''); + + // 兼容：payload.data.list 或 payload.list + $list = []; + if (isset($p['data']['list']) && is_array($p['data']['list'])) $list = $p['data']['list']; + if (!$list && isset($p['list']) && is_array($p['list'])) $list = $p['list']; + + $totalOnPage = (int)($p['totalOnPage'] ?? count($list)); + + return Db::transaction(function () use ($taskId, $page, $serverUrl, $list, $totalOnPage, $p, $now) { + // 1) task + $task = InquiryTask::query()->firstOrCreate( + ['task_id' => $taskId], + [ + 'mode' => 'list', + 'start_page' => $page, + 'max_pages' => 0, + 'status' => 0, + 'server_url' => $serverUrl, + 'meta_json' => null, + 'created_at' => $now, + 'updated_at' => $now, + ] + ); + $task->mode = 'list'; + if ($serverUrl) $task->server_url = $serverUrl; + $task->updated_at = $now; + $task->save(); + + // 2) list_page（唯一：task_id+page） + $pageRow = InquiryListPage::query()->updateOrCreate( + ['task_id' => $taskId, 'page' => $page], + [ + 'total_on_page' => $totalOnPage, + 'is_done' => 1, + 'raw_json' => $this->trimListPageRaw($p), + 'created_at' => $now, + 'updated_at' => $now, + ] + ); + + // 3) list_item（唯一：page_id+idx） + $n = 0; + foreach ($list as $i => $row) { + if (!is_array($row)) continue; + + $idx = (int)($row['idx'] ?? $i); + $detailUrl = (string)($row['detailUrl'] ?? $row['url'] ?? ''); + // $inquiryId = (string)($row['inquiryId'] ?? $row['imInquiryId'] ?? ''); + // ✅ 只信 imInquiryId，避免把 encrypt/hash 长串塞进 inquiry_id + $inquiryId = (string)($row['imInquiryId'] ?? ''); + if (strlen($inquiryId) > 255) $inquiryId = ''; // 兜底 + + + InquiryListItem::query()->updateOrCreate( + ['page_id' => $pageRow->id, 'idx' => $idx], + [ + 'task_id' => $taskId, + 'page' => $page, + 'inquiry_id' => $inquiryId, + 'detail_url' => $detailUrl, + 'buyer_name' => (string)($row['buyerName'] ?? ''), + 'country' => (string)($row['country'] ?? ''), + 'subject' => (string)($row['subject'] ?? ''), + 'snippet' => (string)($row['snippet'] ?? ''), + 'raw_json' => $row, + 'created_at' => $now, + 'updated_at' => $now, + ] + ); + $n++; + } + + return ['ok' => true, 'page' => $page, 'items' => $n, 'listPageId' => (int)$pageRow->id]; + }); + } + + /** 详情上报：写 task + detail_session + detail_message（分块幂等）+（可选）media 元信息绑定 */ + public function handleDetailReport(array $p): array + { + $now = time(); + $taskId = (string)($p['taskId'] ?? ''); + if ($taskId === '') return ['ok' => false, 'err' => 'taskId required']; + + $serverUrl = (string)($p['serverUrl'] ?? ''); + + $detail = $p['detail'] ?? null; + if (!is_array($detail)) return ['ok' => false, 'err' => 'detail required']; + + $detailUrl = (string)($detail['url'] ?? ''); + if ($detailUrl === '') return ['ok' => false, 'err' => 'detail.url required']; + + // $inquiryId = (string)($detail['inquiryId'] ?? $this->extractImInquiryId($detailUrl)); + // ✅ 只从 detailUrl 取 imInquiryId，取不到就空（不要 fallback 到其它字段） + $inquiryId = (string)($this->extractImInquiryId($detailUrl) ?: ''); + if (strlen($inquiryId) > 255) $inquiryId = ''; // 兜底 + + + $customer = $detail['customer'] ?? null; + + $messages = $detail['messages'] ?? []; + if (!is_array($messages)) $messages = []; + + // chunk info + $chunk = $detail['chunk'] ?? []; + $chunkIndex = (int)($chunk['index'] ?? $chunk['seq'] ?? 0); + $isFinal = !empty($chunk['isFinal']); + $messagesCount = (int)($detail['messagesCount'] ?? 0); + + // （可选）如果插件传 listItemId 或者 listRow 的定位信息 + $listItemId = (int)($detail['listItemId'] ?? $p['listItemId'] ?? 0); + + return Db::transaction(function () use ( + $taskId, $serverUrl, $detailUrl, $inquiryId, $customer, + $messages, $chunkIndex, $isFinal, $messagesCount, $listItemId, $detail, $now + ) { + // 1) task + $task = InquiryTask::query()->firstOrCreate( + ['task_id' => $taskId], + [ + 'mode' => 'detail', + 'start_page' => 1, + 'max_pages' => 0, + 'status' => 0, + 'server_url' => $serverUrl, + 'meta_json' => null, + 'created_at' => $now, + 'updated_at' => $now, + ] + ); + $task->mode = 'detail'; + if ($serverUrl) $task->server_url = $serverUrl; + $task->updated_at = $now; + $task->save(); + + // 2) session（唯一：task_id+detail_url） + $session = InquiryDetailSession::query()->firstOrCreate( + ['task_id' => $taskId, 'detail_url' => $detailUrl], + [ + 'list_item_id' => $listItemId, + 'inquiry_id' => $inquiryId, + 'customer_json' => $customer, + 'messages_count' => 0, + 'status' => 0, + 'raw_json' => null, + 'created_at' => $now, + 'updated_at' => $now, + ] + ); + + // 更新 session 头信息 + if ($listItemId && (int)$session->list_item_id === 0) $session->list_item_id = $listItemId; + if ($inquiryId) $session->inquiry_id = $inquiryId; + $session->customer_json = $customer; + $session->raw_json = $this->trimDetailRaw($detail); + $session->updated_at = $now; + $session->save(); + + // 3) messages upsert（唯一：session_id+msg_id） + $upserted = 0; + $mediaBound = 0; + + foreach ($messages as $m) { + if (!is_array($m)) continue; + $msgId = (string)($m['msgId'] ?? ''); + if ($msgId === '') continue; + + $msgRow = InquiryDetailMessage::query()->updateOrCreate( + ['session_id' => $session->id, 'msg_id' => $msgId], + [ + 'direction' => (string)($m['direction'] ?? ''), + 'time_text' => (string)($m['timeText'] ?? ''), + 'original_raw' => (string)($m['originalRaw'] ?? ''), + 'text_shown' => (string)($m['textShown'] ?? ''), + 'translated' => (string)($m['translated'] ?? ''), + 'is_card' => (int)(!empty($m['isCard']) ? 1 : 0), + 'hrefs_json' => $m['hrefs'] ?? [], + 'read_status' => (string)($m['readStatus'] ?? ''), + 'chunk_index' => $chunkIndex, + 'created_at' => $now, + 'updated_at' => $now, + ] + ); + $upserted++; + + // 如果插件在 message 上携带 medias 元信息：先绑定 media 行（文件真正上传由 /media/upload） + if (!empty($m['medias']) && is_array($m['medias'])) { + foreach ($m['medias'] as $media) { + if (!is_array($media)) continue; + $sourceUrl = (string)($media['url'] ?? $media['sourceUrl'] ?? ''); + if ($sourceUrl === '') continue; + + InquiryDetailMedia::query()->updateOrCreate( + [ + 'session_id' => $session->id, + 'message_id' => $msgRow->id, + 'source_url' => $sourceUrl, + ], + [ + 'media_type' => (string)($media['type'] ?? 'file'), + 'file_name' => (string)($media['name'] ?? ''), + 'file_ext' => (string)($media['ext'] ?? ''), + 'size' => (int)($media['size'] ?? 0), + 'md5' => (string)($media['md5'] ?? ''), + 'storage_path' => (string)($media['storagePath'] ?? ''), + 'public_url' => (string)($media['publicUrl'] ?? ''), + 'created_at' => $now, + 'updated_at' => $now, + ] + ); + $mediaBound++; + } + } + } + + // 4) final：统计 messages_count + 标记 session 完成 + if ($isFinal) { + $cnt = InquiryDetailMessage::query()->where('session_id', $session->id)->count(); + $session->messages_count = max($messagesCount, (int)$cnt); + $session->status = 2; // 完成 + $session->updated_at = $now; + $session->save(); + } + + return [ + 'ok' => true, + 'sessionId' => (int)$session->id, + 'chunk' => ['index' => $chunkIndex, 'isFinal' => $isFinal], + 'messagesUpserted' => $upserted, + 'mediasBound' => $mediaBound, + ]; + }); + } + + /** 媒体上传：multipart，保存文件并回填 inquiry_detail_media.storage_path/public_url */ + public function handleUploadMedia(Request $request): array + { + $now = time(); + + $taskId = (string)$request->post('taskId', ''); + $detailUrl = (string)$request->post('detailUrl', ''); + $msgId = (string)$request->post('msgId', ''); + $mediaType = (string)$request->post('mediaType', 'file'); + $sourceUrl = (string)$request->post('sourceUrl', ''); + $md5 = (string)$request->post('md5', ''); + $size = (int)$request->post('size', 0); + $fileName = (string)$request->post('fileName', ''); + + if ($taskId === '' || $detailUrl === '' || $msgId === '') { + return ['ok' => false, 'err' => 'taskId/detailUrl/msgId required']; + } + + $file = $request->file('file'); + if (!$file) return ['ok' => false, 'err' => 'file missing']; + + // 保存目录：runtime/uploads/inquiry_media/YYYYMM/ + $ym = date('Ym'); + $dir = base_path() . "/runtime/uploads/inquiry_media/{$ym}"; + if (!is_dir($dir)) @mkdir($dir, 0777, true); + + $originalName = $fileName ?: ($file->getUploadName() ?: ('media_' . $now)); + $ext = strtolower(pathinfo($originalName, PATHINFO_EXTENSION)); + $safeExt = $ext ? preg_replace('/[^a-z0-9]/i', '', $ext) : ''; + + $saveName = 't' . $taskId . '_' . md5($originalName . $now . random_int(1, 999999)) . ($safeExt ? ".{$safeExt}" : ''); + $file->move($dir . '/' . $saveName); + + // 注意：serveMedia 是从 runtime/uploads/ 下面读，所以这里用相对 uploads 的路径 + // full: runtime/uploads/inquiry_media/YYYYMM/xxx.ext + $storageRel = "inquiry_media/{$ym}/{$saveName}"; + $storagePath = "/runtime/uploads/{$storageRel}"; + $publicUrl = "/media/{$storageRel}"; + + return Db::transaction(function () use ( + $taskId, $detailUrl, $msgId, $mediaType, $sourceUrl, $md5, $size, $originalName, $safeExt, + $storagePath, $publicUrl, $now + ) { + // session：确保存在 + $session = InquiryDetailSession::query()->firstOrCreate( + ['task_id' => $taskId, 'detail_url' => $detailUrl], + [ + 'list_item_id' => 0, + 'inquiry_id' => $this->extractImInquiryId($detailUrl), + 'customer_json' => null, + 'messages_count' => 0, + 'status' => 0, + 'raw_json' => null, + 'created_at' => $now, + 'updated_at' => $now, + ] + ); + + // message：确保存在（某些媒体消息可能先传媒体再传 message） + $msgRow = InquiryDetailMessage::query()->firstOrCreate( + ['session_id' => $session->id, 'msg_id' => $msgId], + [ + 'direction' => '', + 'time_text' => '', + 'original_raw' => '', + 'text_shown' => '', + 'translated' => '', + 'is_card' => 1, + 'hrefs_json' => [], + 'read_status' => '', + 'chunk_index' => 0, + 'created_at' => $now, + 'updated_at' => $now, + ] + ); + + // media：幂等（唯一：session_id+message_id+source_url） + $uniqSource = $sourceUrl ?: $storagePath; + + $media = InquiryDetailMedia::query()->updateOrCreate( + ['session_id' => $session->id, 'message_id' => $msgRow->id, 'source_url' => $uniqSource], + [ + 'media_type' => $mediaType, + 'source_url' => $uniqSource, + 'file_name' => $originalName, + 'file_ext' => $safeExt, + 'size' => $size, + 'md5' => $md5, + 'storage_path' => $storagePath, + 'public_url' => $publicUrl, + 'created_at' => $now, + 'updated_at' => $now, + ] + ); + + return [ + 'ok' => true, + 'mediaId' => (int)$media->id, + 'storage_path' => $storagePath, + 'public_url' => $publicUrl, + 'file_name' => $originalName, + ]; + }); + } + + /** 本地媒体访问：GET /media/{path} -> runtime/uploads/{path} */ + public function serveMediaFile(string $path): Response + { + // 防路径穿越 + $path = str_replace(['..', '\\'], ['', '/'], $path); + $full = base_path() . '/runtime/uploads/' . $path; + if (!is_file($full)) { + return response('Not Found', 404); + } + + $mime = $this->guessMime($full); + return response(file_get_contents($full), 200, ['Content-Type' => $mime]); + } + + private function guessMime(string $file): string + { + $ext = strtolower(pathinfo($file, PATHINFO_EXTENSION)); + return match ($ext) { + 'jpg','jpeg' => 'image/jpeg', + 'png' => 'image/png', + 'gif' => 'image/gif', + 'webp' => 'image/webp', + 'mp4' => 'video/mp4', + 'pdf' => 'application/pdf', + 'xls' => 'application/vnd.ms-excel', + 'xlsx' => 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + 'doc' => 'application/msword', + 'docx' => 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'ppt' => 'application/vnd.ms-powerpoint', + 'pptx' => 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + 'zip' => 'application/zip', + default => 'application/octet-stream' + }; + } + + private function extractImInquiryId(string $url): string + { + if ($url === '') return ''; + $m = []; + if (preg_match('/[?&]imInquiryId=([^&]+)/i', $url, $m)) return (string)$m[1]; + return ''; + } + + /** 裁剪 list_page raw_json（避免存超大） */ + private function trimListPageRaw(array $p): array + { + $keep = $p; + if (isset($keep['list'])) unset($keep['list']); + if (isset($keep['data']['list'])) unset($keep['data']['list']); + return $keep; + } + + /** 裁剪 detail raw_json（避免存超大 messages） */ + private function trimDetailRaw(array $detail): array + { + $keep = $detail; + if (isset($keep['messages'])) unset($keep['messages']); + return $keep; + } +} diff --git a/config/route.php b/config/route.php index fd6c729..d4316a6 100644 --- a/config/route.php +++ b/config/route.php @@ -19,3 +19,16 @@ use Webman\Route; Route::any('/ts/form', [app\controller\TransmitController::class, 'form']); Route::any('/iq/form', [app\inquery\controller\InqueryController::class, 'form']); + + + +Route::any('/inquiry/list/report', [app\alibaba\controller\InquiryScraperController::class, 'reportList']); +Route::any('/inquiry/detail/report', [app\alibaba\controller\InquiryScraperController::class, 'reportDetail']); +Route::any('/inquiry/detail/media/upload', [app\alibaba\controller\InquiryScraperController::class, 'uploadMedia']); + +// 可选：本地媒体访问（插件上传后可直接访问） +Route::any('/media/{path:.+}', [app\alibaba\controller\InquiryScraperController::class, 'serveMedia']); + + + +Route::any('/inquiry/contact/report', [app\alibaba\controller\InquiryScraperController::class, 'reportContact']); \ No newline at end of file