|
|
@@ -1,8 +1,14 @@
|
|
|
'use strict';
|
|
|
|
|
|
// 采集引擎：按店铺批量抓取菜品原图。
|
|
|
-// 从 server.js 原样搬入：会话式 invokeApi、翻页、下载、断点跳过、店铺完成状态、可取消的任务循环。
|
|
|
+// 从 server.js 原样搬入：翻页、下载、断点跳过、店铺完成状态、可取消的任务循环。
|
|
|
// 状态（crawlState/cancelFlag）收敛在本工厂内，通过返回的接口访问。
|
|
|
+//
|
|
|
+// 采集通道（v2）：
|
|
|
+// · client —— 请求放进淘宝闪购客户端页面上下文发出（CDP :9669），登录态随客户端走，无需 Cookie；
|
|
|
+// · cookie —— 原直连通道，靠手填 Cookie（ksid）鉴权，保留作为兜底；
|
|
|
+// · auto —— 优先客户端，客户端不可用/中途失联时自动回落 Cookie。
|
|
|
+// 两通道的请求体/响应形状完全一致，仅传输层不同；图片文件下载不走通道（CDN 直下，无需登录态）。
|
|
|
|
|
|
const fs = require('fs');
|
|
|
const path = require('path');
|
|
|
@@ -11,6 +17,12 @@ const https = require('https');
|
|
|
const http = require('http');
|
|
|
const { URL } = require('url');
|
|
|
const { sendJson, sleep, sanitizeFileName, guessExtFromUrl, SKIP_DIR_NAMES } = require('../util.js');
|
|
|
+const kit = require('../../napos/kit.js');
|
|
|
+
|
|
|
+const GOODS_INVOKE_PATH = '/nevermore.goods/invoke';
|
|
|
+const GOODS_SERVICE = 'ItemPhotoProcessService';
|
|
|
+const GOODS_METHOD = 'getItemPhotoGallery';
|
|
|
+const MAX_PAGES = 200;
|
|
|
|
|
|
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
|
|
|
|
|
@@ -19,6 +31,9 @@ function createCrawlService(ctx) {
|
|
|
|
|
|
let crawlState = { status: 'idle', startedAt: null, finishedAt: null, shops: [], log: [] };
|
|
|
let cancelFlag = false;
|
|
|
+ // 通道：channelMode 是用户选择（auto/client/cookie）；useClient 是本次运行实际生效通道（auto 下可中途回落）
|
|
|
+ let channelMode = 'auto';
|
|
|
+ let useClient = false;
|
|
|
|
|
|
// ---------- 采集状态持久化（重启后进度不丢） ----------
|
|
|
|
|
|
@@ -46,6 +61,7 @@ function createCrawlService(ctx) {
|
|
|
startedAt: saved.startedAt || null,
|
|
|
finishedAt: saved.finishedAt || null,
|
|
|
forceFullCrawl: !!saved.forceFullCrawl,
|
|
|
+ channel: saved.channel || null,
|
|
|
shops: Array.isArray(saved.shops) ? saved.shops : [],
|
|
|
log: Array.isArray(saved.log) ? saved.log : [],
|
|
|
};
|
|
|
@@ -66,20 +82,27 @@ function createCrawlService(ctx) {
|
|
|
if (crawlState.log.length > 500) crawlState.log.shift();
|
|
|
}
|
|
|
|
|
|
- function invokeApi(service, method, requestParams, extraMetas) {
|
|
|
+ /** 两通道共用的请求体（与真实抓包一致：ksid 在 metas 里，一次性 id） */
|
|
|
+ function buildRequestBody(service, method, requestParams, ksid, extraMetas) {
|
|
|
+ return {
|
|
|
+ id: crypto.randomBytes(16).toString('hex').toUpperCase() + '|' + Date.now(),
|
|
|
+ method, service,
|
|
|
+ params: { request: requestParams || {} },
|
|
|
+ metas: Object.assign({ appName: 'melody', appVersion: '4.4.0', ksid }, extraMetas || {}),
|
|
|
+ ncp: '2.0.0', adapter: 'ncp',
|
|
|
+ extendConfig: { docMode: 'master,,' },
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ // ---- Cookie 直连通道（原实现，兜底保留） ----
|
|
|
+
|
|
|
+ async function httpInvoke(service, method, requestParams, extraMetas) {
|
|
|
return new Promise((resolve, reject) => {
|
|
|
- const body = {
|
|
|
- id: crypto.randomBytes(16).toString('hex').toUpperCase() + '|' + Date.now(),
|
|
|
- method, service,
|
|
|
- params: { request: requestParams || {} },
|
|
|
- metas: Object.assign({ appName: 'melody', appVersion: '4.4.0', ksid: session.getKsid() }, extraMetas || {}),
|
|
|
- ncp: '2.0.0', adapter: 'ncp',
|
|
|
- extendConfig: { docMode: 'master,,' },
|
|
|
- };
|
|
|
+ const body = buildRequestBody(service, method, requestParams, session.getKsid(), extraMetas);
|
|
|
const data = JSON.stringify(body);
|
|
|
const options = {
|
|
|
hostname: 'app-api.shop.ele.me', port: 443,
|
|
|
- path: `/nevermore.goods/invoke?method=${encodeURIComponent(service + '.' + method)}`,
|
|
|
+ path: `${GOODS_INVOKE_PATH}?method=${encodeURIComponent(service + '.' + method)}`,
|
|
|
method: 'POST',
|
|
|
headers: {
|
|
|
'Content-Type': 'application/json; charset=UTF-8', Accept: 'application/json',
|
|
|
@@ -106,6 +129,109 @@ function createCrawlService(ctx) {
|
|
|
});
|
|
|
}
|
|
|
|
|
|
+ // ---- 客户端通道（CDP）：请求由客户端页面上下文发出，Cookie/UA 全是真 ----
|
|
|
+
|
|
|
+ function makeClientInvoker(api, ksid) {
|
|
|
+ return async function (service, method, requestParams, extraMetas) {
|
|
|
+ const url = `https://app-api.shop.ele.me${GOODS_INVOKE_PATH}?method=${encodeURIComponent(service + '.' + method)}`;
|
|
|
+ const res = await kit.cdp.pageFetch(api, {
|
|
|
+ method: 'POST', url, allowWrite: false,
|
|
|
+ body: buildRequestBody(service, method, requestParams, ksid, extraMetas),
|
|
|
+ });
|
|
|
+ if (res.error || res.timeout) {
|
|
|
+ throw new Error('客户端内请求失败: ' + (res.error || '超时（可能被风控挂起，请在客户端检查）'));
|
|
|
+ }
|
|
|
+ return {
|
|
|
+ statusCode: res.status,
|
|
|
+ json: res.parsed ? res.data : null,
|
|
|
+ raw: res.parsed ? '' : String(res.data || ''),
|
|
|
+ risk: !!res.risk, authFail: !!res.authFail,
|
|
|
+ };
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 翻页拉全一个店铺的菜品列表（不含下载）。
|
|
|
+ * @param {Function} invoke (service, method, requestParams, extraMetas) => resp，由通道提供
|
|
|
+ * @returns {{items:Array, authFailed:boolean, risk:boolean, error:?string}}
|
|
|
+ * error 只作记录不中断：已拉到的部分照常下载（Cookie 通道保留原行为）
|
|
|
+ */
|
|
|
+ async function fetchAllPages(shopEntry, invoke, pageSize) {
|
|
|
+ const allItems = [];
|
|
|
+ const seen = new Set();
|
|
|
+ let nextId;
|
|
|
+ let page = 0;
|
|
|
+ let authFailed = false;
|
|
|
+ let risk = false;
|
|
|
+ let error = null;
|
|
|
+ while (page < MAX_PAGES) {
|
|
|
+ if (cancelFlag) break;
|
|
|
+ page++;
|
|
|
+ const requestParams = {
|
|
|
+ shopId: shopEntry.shopId, pageSize, queryType: 'SHOP',
|
|
|
+ searchKey: '', searchType: ['ITEM_NAME'],
|
|
|
+ };
|
|
|
+ if (nextId) requestParams.nextId = nextId;
|
|
|
+
|
|
|
+ let resp;
|
|
|
+ try { resp = await invoke(GOODS_SERVICE, GOODS_METHOD, requestParams, { shopId: shopEntry.shopId }); }
|
|
|
+ catch (e) {
|
|
|
+ error = '请求失败: ' + (e.message || e);
|
|
|
+ pushLog(`[${shopEntry.shopName}] ${error}`);
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ if (resp.risk || resp.authFail || isAuthError(resp)) {
|
|
|
+ authFailed = true;
|
|
|
+ risk = !!resp.risk;
|
|
|
+ pushLog(`[${shopEntry.shopName}] ${risk ? '触发风控/验证码' : '鉴权失败'}（登录态可能已失效）`);
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ const items = (resp.json && resp.json.result && resp.json.result.result) || [];
|
|
|
+ nextId = resp.json && resp.json.result && resp.json.result.nextId;
|
|
|
+ if (!items.length) {
|
|
|
+ pushLog(`[${shopEntry.shopName}] 第${page}页无数据，结束翻页`);
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ let newCount = 0;
|
|
|
+ for (const it of items) {
|
|
|
+ const gid = it.itemGlobalId;
|
|
|
+ if (gid !== undefined && gid !== null && seen.has(gid)) continue;
|
|
|
+ if (gid !== undefined && gid !== null) seen.add(gid);
|
|
|
+ allItems.push(it);
|
|
|
+ newCount++;
|
|
|
+ }
|
|
|
+ pushLog(`[${shopEntry.shopName}] 第${page}页 新增${newCount}条`);
|
|
|
+ if (!nextId) break;
|
|
|
+ await sleep(200);
|
|
|
+ }
|
|
|
+ return { items: allItems, authFailed, risk, error };
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 客户端通道拉取：每店一个 CDP 会话（复用于本店全部翻页），
|
|
|
+ * 连接坏了只损失当前店，下一店自动重连。
|
|
|
+ * @returns 页结果；null 表示应回落 Cookie 通道；shopFailed=true 表示本店已判失败（无 Cookie 可回落）
|
|
|
+ */
|
|
|
+ async function fetchViaClient(shopEntry, pageSize) {
|
|
|
+ try {
|
|
|
+ return await kit.cdp.withPageFor(kit.DEFAULT_PORT, 'shop', async (api) => {
|
|
|
+ const ksid = await kit.cdp.findKsid(api);
|
|
|
+ if (!ksid) throw new Error('客户端未登录（读不到登录态 ksid），请先在客户端登录');
|
|
|
+ return fetchAllPages(shopEntry, makeClientInvoker(api, ksid), pageSize);
|
|
|
+ }, { needExact: false, autoOpen: false });
|
|
|
+ } catch (e) {
|
|
|
+ const msg = e.message || String(e);
|
|
|
+ if (!(channelMode === 'auto' && session.cookie && session.getKsid())) {
|
|
|
+ shopEntry.state = 'failed';
|
|
|
+ shopEntry.error = '客户端通道失败: ' + msg;
|
|
|
+ pushLog(`[${shopEntry.shopName}] ${shopEntry.error}`);
|
|
|
+ return { items: [], authFailed: false, risk: false, error: shopEntry.error, shopFailed: true };
|
|
|
+ }
|
|
|
+ pushLog(`[${shopEntry.shopName}] 客户端通道失败: ${msg}`);
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
function isAuthError(resp) {
|
|
|
if (resp.statusCode === 401 || resp.statusCode === 403) return true;
|
|
|
const j = resp.json;
|
|
|
@@ -190,7 +316,22 @@ function createCrawlService(ctx) {
|
|
|
|
|
|
async function startCrawl(body, res) {
|
|
|
if (crawlState.status === 'running') return sendJson(res, 409, { error: '已有抓取任务在进行中' });
|
|
|
- if (!session.cookie || !session.getKsid()) return sendJson(res, 400, { error: '请先填写包含 ksid 的 Cookie' });
|
|
|
+ // 通道解析：body.channel（单次指定）> config.crawlChannel（设置）> auto
|
|
|
+ const channelPref = ['auto', 'client', 'cookie'].includes(body && body.channel) ? body.channel : (config.crawlChannel || 'auto');
|
|
|
+ const cookieOk = !!(session.cookie && session.getKsid());
|
|
|
+ const probeResult = await kit.cdp.probe().catch(() => ({ ok: false }));
|
|
|
+ const clientOk = !!probeResult.ok;
|
|
|
+ if (channelPref === 'client' && !clientOk) {
|
|
|
+ return sendJson(res, 400, { error: '客户端通道不可用：请先启动淘宝闪购客户端（调试端口 9669），或改用 Cookie 通道' });
|
|
|
+ }
|
|
|
+ if (channelPref === 'cookie' && !cookieOk) {
|
|
|
+ return sendJson(res, 400, { error: '请先填写包含 ksid 的 Cookie（设置 → 采集配置）' });
|
|
|
+ }
|
|
|
+ if (channelPref === 'auto' && !clientOk && !cookieOk) {
|
|
|
+ return sendJson(res, 400, { error: '未检测到客户端（调试端口 9669 未开），也未填写 Cookie——两种通道至少配置其一' });
|
|
|
+ }
|
|
|
+ channelMode = channelPref;
|
|
|
+ useClient = channelPref === 'client' || (channelPref === 'auto' && clientOk);
|
|
|
const root = String((body && body.root) || workspace.getWorkspaceRoot()).trim();
|
|
|
if (!workspace.isWorkspaceRoot(root)) return sendJson(res, 403, { error: '仅允许采集当前输出根目录' });
|
|
|
if (!fs.existsSync(root) || !fs.statSync(root).isDirectory()) {
|
|
|
@@ -210,6 +351,7 @@ function createCrawlService(ctx) {
|
|
|
crawlState = {
|
|
|
status: pendingShops.length ? 'running' : 'done', startedAt: Date.now(),
|
|
|
finishedAt: pendingShops.length ? null : Date.now(), forceFullCrawl,
|
|
|
+ channel: useClient ? 'client' : 'cookie',
|
|
|
shops: [
|
|
|
...skippedCompleted.map((s) => ({
|
|
|
shopId: s.shopId, shopName: s.shopName, folderName: completed[String(s.shopId)].folderName || null,
|
|
|
@@ -223,7 +365,7 @@ function createCrawlService(ctx) {
|
|
|
],
|
|
|
log: [],
|
|
|
};
|
|
|
- pushLog(`开始抓取，实际抓取 ${pendingShops.length} 家${skippedCompleted.length ? `，历史完成跳过 ${skippedCompleted.length} 家` : ''}，root=${root}`);
|
|
|
+ pushLog(`开始抓取，实际抓取 ${pendingShops.length} 家${skippedCompleted.length ? `，历史完成跳过 ${skippedCompleted.length} 家` : ''}，root=${root}，通道=${useClient ? '客户端（CDP :9669）' : 'Cookie 直连'}${channelPref === 'auto' ? '（自动）' : ''}`);
|
|
|
persistState();
|
|
|
sendJson(res, 200, { ok: true, crawlCount: pendingShops.length, skippedCompletedCount: skippedCompleted.length });
|
|
|
if (!pendingShops.length) return pushLog('所选店铺均已完成；勾选"强制全量抓取"可重新下载');
|
|
|
@@ -259,57 +401,29 @@ function createCrawlService(ctx) {
|
|
|
shopEntry.folderName = dirInfo.folderName;
|
|
|
pushLog(`[${shopEntry.shopName}] ${dirInfo.reused ? '复用已有目录' : '新建目录'} ${dirInfo.folderName}`);
|
|
|
|
|
|
- const allItems = [];
|
|
|
- const seen = new Set();
|
|
|
- let nextId;
|
|
|
- let page = 0;
|
|
|
- const MAX_PAGES = 200;
|
|
|
- let authFailed = false;
|
|
|
-
|
|
|
- while (page < MAX_PAGES) {
|
|
|
- if (cancelFlag) break;
|
|
|
- page++;
|
|
|
- const requestParams = {
|
|
|
- shopId: shopEntry.shopId, pageSize, queryType: 'SHOP',
|
|
|
- searchKey: '', searchType: ['ITEM_NAME'],
|
|
|
- };
|
|
|
- if (nextId) requestParams.nextId = nextId;
|
|
|
-
|
|
|
- let resp;
|
|
|
- try { resp = await invokeApi('ItemPhotoProcessService', 'getItemPhotoGallery', requestParams, { shopId: shopEntry.shopId }); }
|
|
|
- catch (e) {
|
|
|
- shopEntry.error = '请求失败: ' + (e.message || e);
|
|
|
- pushLog(`[${shopEntry.shopName}] 请求失败: ${e.message || e}`);
|
|
|
- break;
|
|
|
- }
|
|
|
- if (isAuthError(resp)) {
|
|
|
- authFailed = true;
|
|
|
- shopEntry.error = '会话可能已过期，请重新登录后更新 Cookie';
|
|
|
- pushLog(`[${shopEntry.shopName}] 鉴权失败（ksid/Cookie 可能已过期）`);
|
|
|
- break;
|
|
|
- }
|
|
|
- const items = (resp.json && resp.json.result && resp.json.result.result) || [];
|
|
|
- nextId = resp.json && resp.json.result && resp.json.result.nextId;
|
|
|
- if (!items.length) {
|
|
|
- pushLog(`[${shopEntry.shopName}] 第${page}页无数据，结束翻页`);
|
|
|
- break;
|
|
|
- }
|
|
|
- let newCount = 0;
|
|
|
- for (const it of items) {
|
|
|
- const gid = it.itemGlobalId;
|
|
|
- if (gid !== undefined && gid !== null && seen.has(gid)) continue;
|
|
|
- if (gid !== undefined && gid !== null) seen.add(gid);
|
|
|
- allItems.push(it);
|
|
|
- newCount++;
|
|
|
+ // ---- 翻页：按生效通道拉取（客户端优先，可回落 Cookie） ----
|
|
|
+ let pageResult = null;
|
|
|
+ if (useClient) {
|
|
|
+ pageResult = await fetchViaClient(shopEntry, pageSize);
|
|
|
+ if (pageResult === null) {
|
|
|
+ useClient = false;
|
|
|
+ crawlState.channel = 'cookie';
|
|
|
+ pushLog('本店起改用 Cookie 通道继续');
|
|
|
+ } else if (pageResult.shopFailed) {
|
|
|
+ persistState();
|
|
|
+ continue;
|
|
|
}
|
|
|
- pushLog(`[${shopEntry.shopName}] 第${page}页 新增${newCount}条`);
|
|
|
- if (!nextId) break;
|
|
|
- await sleep(200);
|
|
|
}
|
|
|
+ if (!pageResult) pageResult = await fetchAllPages(shopEntry, httpInvoke, pageSize);
|
|
|
|
|
|
+ const allItems = pageResult.items;
|
|
|
shopEntry.total = allItems.length;
|
|
|
- if (authFailed && !allItems.length) {
|
|
|
+ if (pageResult.error && !pageResult.authFailed) shopEntry.error = pageResult.error;
|
|
|
+ if (pageResult.authFailed && !allItems.length) {
|
|
|
shopEntry.state = 'failed';
|
|
|
+ shopEntry.error = pageResult.risk
|
|
|
+ ? (useClient ? '触发风控/验证码，请在客户端完成验证后重试' : '触发风控，请稍后重试或更新 Cookie')
|
|
|
+ : (useClient ? '客户端登录态失效，请在客户端重新登录后重试' : '会话可能已过期，请重新登录后更新 Cookie');
|
|
|
continue;
|
|
|
}
|
|
|
|
