hyreal-gen-pic/server/services/crawl.js

'use strict';

// 采集引擎：按店铺批量抓取菜品原图。
// 从 server.js 原样搬入：会话式 invokeApi、翻页、下载、断点跳过、店铺完成状态、可取消的任务循环。
// 状态（crawlState/cancelFlag）收敛在本工厂内，通过返回的接口访问。

const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const https = require('https');
const http = require('http');
const { URL } = require('url');
const { sendJson, sleep, sanitizeFileName, guessExtFromUrl, SKIP_DIR_NAMES } = require('../util.js');

const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';

function createCrawlService(ctx) {
  const { config, session, shops, workspace, logs } = ctx;

  let crawlState = { status: 'idle', startedAt: null, finishedAt: null, shops: [], log: [] };
  let cancelFlag = false;

  // ---------- 采集状态持久化（重启后进度不丢） ----------

  const stateFilePath = () => path.join(workspace.getWorkspaceRoot(), '.genpic', 'crawl-state.json');

  function persistState() {
    if (crawlState.status === 'idle' && !crawlState.shops.length) return;
    try {
      const file = stateFilePath();
      fs.mkdirSync(path.dirname(file), { recursive: true });
      const temp = `${file}.${process.pid}.tmp`;
      const snapshot = { ...crawlState, log: crawlState.log.slice(-100) };
      fs.writeFileSync(temp, JSON.stringify({ version: 1, savedAt: new Date().toISOString(), state: snapshot }, null, 2), 'utf8');
      fs.renameSync(temp, file);
    } catch { /* 持久化失败不影响采集进行 */ }
  }

  function restoreState() {
    let parsed;
    try { parsed = JSON.parse(fs.readFileSync(stateFilePath(), 'utf8')); } catch { return; }
    const saved = parsed && parsed.version === 1 && parsed.state;
    if (!saved || typeof saved.status !== 'string') return;
    crawlState = {
      status: saved.status,
      startedAt: saved.startedAt || null,
      finishedAt: saved.finishedAt || null,
      forceFullCrawl: !!saved.forceFullCrawl,
      shops: Array.isArray(saved.shops) ? saved.shops : [],
      log: Array.isArray(saved.log) ? saved.log : [],
    };
    if (crawlState.status === 'running') {
      crawlState.status = 'interrupted';
      crawlState.finishedAt = Date.now();
      for (const s of crawlState.shops) {
        if (s.state === 'running' || s.state === 'pending') s.state = 'interrupted';
      }
      crawlState.log.push(logs.createLogEntry('服务重启，上次采集被中断；未完成的店铺可重新勾选采集', 'task', 'warn'));
    }
  }
  restoreState();

  function pushLog(msg) {
    const entry = logs.createLogEntry(msg, 'task', logs.detectLogLevel(msg));
    crawlState.log.push(entry);
    if (crawlState.log.length > 500) crawlState.log.shift();
  }

  function invokeApi(service, method, requestParams, extraMetas) {
    return new Promise((resolve, reject) => {
      const body = {
        id: crypto.randomBytes(16).toString('hex').toUpperCase() + '|' + Date.now(),
        method, service,
        params: { request: requestParams || {} },
        metas: Object.assign({ appName: 'melody', appVersion: '4.4.0', ksid: session.getKsid() }, extraMetas || {}),
        ncp: '2.0.0', adapter: 'ncp',
        extendConfig: { docMode: 'master,,' },
      };
      const data = JSON.stringify(body);
      const options = {
        hostname: 'app-api.shop.ele.me', port: 443,
        path: `/nevermore.goods/invoke?method=${encodeURIComponent(service + '.' + method)}`,
        method: 'POST',
        headers: {
          'Content-Type': 'application/json; charset=UTF-8', Accept: 'application/json',
          Origin: 'https://napos-goods-pc.faas.ele.me', Referer: 'https://napos-goods-pc.faas.ele.me/',
          Cookie: session.cookie || '', 'User-Agent': UA,
          'Content-Length': Buffer.byteLength(data),
        },
        timeout: config.requestTimeoutMs,
      };
      const req = https.request(options, (res) => {
        const chunks = [];
        res.on('data', (c) => chunks.push(c));
        res.on('end', () => {
          const raw = Buffer.concat(chunks).toString('utf8');
          let json = null;
          try { json = JSON.parse(raw); } catch { /* keep raw */ }
          resolve({ statusCode: res.statusCode, json, raw });
        });
      });
      req.on('error', reject);
      req.on('timeout', () => req.destroy(new Error('请求超时')));
      req.write(data);
      req.end();
    });
  }

  function isAuthError(resp) {
    if (resp.statusCode === 401 || resp.statusCode === 403) return true;
    const j = resp.json;
    if (!j) return false;
    if (j.result !== undefined) return false;
    return /未登录|请登录|login|unauthorized|token.*expired|session.*expired/i.test(JSON.stringify(j));
  }

  function downloadImage(urlStr, redirectsLeft) {
    if (redirectsLeft === undefined) redirectsLeft = 5;
    return new Promise((resolve, reject) => {
      let u;
      try { u = new URL(urlStr); } catch (e) { return reject(new Error('无效的图片地址: ' + urlStr)); }
      const mod = u.protocol === 'http:' ? http : https;
      mod.get(u, { headers: { Referer: 'https://melody.shop.ele.me/', 'User-Agent': UA } }, (res) => {
        if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location && redirectsLeft > 0) {
          res.resume();
          return resolve(downloadImage(res.headers.location, redirectsLeft - 1));
        }
        if (res.statusCode !== 200) { res.resume(); return reject(new Error('下载图片失败 HTTP ' + res.statusCode)); }
        const chunks = [];
        res.on('data', (c) => chunks.push(c));
        res.on('end', () => resolve(Buffer.concat(chunks)));
      }).on('error', reject);
    });
  }

  // ---------- 店铺完成状态（root 下 _膳绘_店铺状态.json） ----------

  function completionStatePath(root) { return path.join(root, '_膳绘_店铺状态.json'); }

  function readCompletionState(root) {
    try {
      const data = JSON.parse(fs.readFileSync(completionStatePath(root), 'utf8'));
      return data.version === 1 && data.shops && typeof data.shops === 'object' ? data.shops : {};
    } catch { return {}; }
  }

  function writeCompletionState(root, shopsState) {
    const target = completionStatePath(root);
    const temp = `${target}.${process.pid}.tmp`;
    fs.writeFileSync(temp, JSON.stringify({ version: 1, shops: shopsState }, null, 2), 'utf8');
    fs.renameSync(temp, target);
  }

  function markShopCompleted(root, shopEntry) {
    const shopsState = readCompletionState(root);
    shopsState[String(shopEntry.shopId)] = {
      shopId: shopEntry.shopId, shopName: shopEntry.shopName, folderName: shopEntry.folderName,
      completedAt: new Date().toISOString(), downloaded: shopEntry.downloaded,
      skipped: shopEntry.skipped, total: shopEntry.total,
    };
    writeCompletionState(root, shopsState);
  }

  // ---------- 门店目录分配 ----------

  function listRootDirs(root) {
    return fs.readdirSync(root, { withFileTypes: true })
      .filter((e) => e.isDirectory() && !e.name.startsWith('.') && !SKIP_DIR_NAMES.has(e.name))
      .map((e) => e.name);
  }

  function resolveShopDir(root, shopId, shopName) {
    const dirs = listRootDirs(root);
    const suffix = '_' + String(shopId);
    const match = dirs.find((d) => d.endsWith(suffix));
    if (match) return { dir: path.join(root, match), folderName: match, reused: true };
    let maxSeq = 0;
    for (const d of dirs) {
      const mm = /^(\d+)_/.exec(d);
      if (mm) maxSeq = Math.max(maxSeq, parseInt(mm[1], 10));
    }
    const seq = String(maxSeq + 1).padStart(3, '0');
    const folderName = `${seq}_${sanitizeFileName(shopName)}_${shopId}`;
    const dir = path.join(root, folderName);
    fs.mkdirSync(dir, { recursive: true });
    return { dir, folderName, reused: false };
  }

  // ---------- 任务循环 ----------

  async function startCrawl(body, res) {
    if (crawlState.status === 'running') return sendJson(res, 409, { error: '已有抓取任务在进行中' });
    if (!session.cookie || !session.getKsid()) return sendJson(res, 400, { error: '请先填写包含 ksid 的 Cookie' });
    const root = String((body && body.root) || workspace.getWorkspaceRoot()).trim();
    if (!workspace.isWorkspaceRoot(root)) return sendJson(res, 403, { error: '仅允许采集当前输出根目录' });
    if (!fs.existsSync(root) || !fs.statSync(root).isDirectory()) {
      return sendJson(res, 400, { error: '输出根目录不存在: ' + root });
    }
    const shopIds = Array.isArray(body && body.shopIds) ? body.shopIds.map(Number) : [];
    const selectedShops = shopIds.map((id) => shops.getList().find((s) => s.shopId === id)).filter(Boolean);
    if (!selectedShops.length) return sendJson(res, 400, { error: '没有可抓取的店铺，请先填写手动店铺清单并勾选' });
    const forceFullCrawl = Boolean(body && body.forceFullCrawl);
    const completed = readCompletionState(root);
    const skippedCompleted = forceFullCrawl ? [] : selectedShops.filter((s) => completed[String(s.shopId)]);
    const pendingShops = forceFullCrawl ? selectedShops : selectedShops.filter((s) => !completed[String(s.shopId)]);
    const pageSize = Number(body && body.pageSize) || config.pageSize;
    const delayMs = Number(body && body.delayMs) || config.delayMs;

    cancelFlag = false;
    crawlState = {
      status: pendingShops.length ? 'running' : 'done', startedAt: Date.now(),
      finishedAt: pendingShops.length ? null : Date.now(), forceFullCrawl,
      shops: [
        ...skippedCompleted.map((s) => ({
          shopId: s.shopId, shopName: s.shopName, folderName: completed[String(s.shopId)].folderName || null,
          state: 'skipped_completed', total: completed[String(s.shopId)].total || 0,
          downloaded: 0, skipped: 0, failed: 0, error: null,
        })),
        ...pendingShops.map((s) => ({
          shopId: s.shopId, shopName: s.shopName, folderName: null, state: 'pending',
          total: 0, downloaded: 0, skipped: 0, failed: 0, error: null,
        })),
      ],
      log: [],
    };
    pushLog(`开始抓取，实际抓取 ${pendingShops.length} 家${skippedCompleted.length ? `，历史完成跳过 ${skippedCompleted.length} 家` : ''}，root=${root}`);
    persistState();
    sendJson(res, 200, { ok: true, crawlCount: pendingShops.length, skippedCompletedCount: skippedCompleted.length });
    if (!pendingShops.length) return pushLog('所选店铺均已完成；勾选"强制全量抓取"可重新下载');

    runCrawlLoop(root, pageSize, delayMs, forceFullCrawl).catch((e) => {
      crawlState.status = 'error';
      crawlState.finishedAt = Date.now();
      pushLog('抓取流程异常终止: ' + (e.message || e));
      persistState();
    });
  }

  async function runCrawlLoop(root, pageSize, delayMs, forceFullCrawl) {
    for (const shopEntry of crawlState.shops) {
      if (shopEntry.state === 'skipped_completed') continue;
      if (cancelFlag) {
        crawlState.status = 'stopped';
        crawlState.finishedAt = Date.now();
        pushLog('已停止');
        return;
      }
      shopEntry.state = 'running';
      pushLog(`[${shopEntry.shopName}] 开始`);

      let dirInfo;
      try { dirInfo = resolveShopDir(root, shopEntry.shopId, shopEntry.shopName); }
      catch (e) {
        shopEntry.state = 'failed';
        shopEntry.error = '创建目录失败: ' + (e.message || e);
        pushLog(`[${shopEntry.shopName}] 创建目录失败: ${e.message || e}`);
        continue;
      }
      shopEntry.folderName = dirInfo.folderName;
      pushLog(`[${shopEntry.shopName}] ${dirInfo.reused ? '复用已有目录' : '新建目录'} ${dirInfo.folderName}`);

      const allItems = [];
      const seen = new Set();
      let nextId;
      let page = 0;
      const MAX_PAGES = 200;
      let authFailed = false;

      while (page < MAX_PAGES) {
        if (cancelFlag) break;
        page++;
        const requestParams = {
          shopId: shopEntry.shopId, pageSize, queryType: 'SHOP',
          searchKey: '', searchType: ['ITEM_NAME'],
        };
        if (nextId) requestParams.nextId = nextId;

        let resp;
        try { resp = await invokeApi('ItemPhotoProcessService', 'getItemPhotoGallery', requestParams, { shopId: shopEntry.shopId }); }
        catch (e) {
          shopEntry.error = '请求失败: ' + (e.message || e);
          pushLog(`[${shopEntry.shopName}] 请求失败: ${e.message || e}`);
          break;
        }
        if (isAuthError(resp)) {
          authFailed = true;
          shopEntry.error = '会话可能已过期，请重新登录后更新 Cookie';
          pushLog(`[${shopEntry.shopName}] 鉴权失败（ksid/Cookie 可能已过期）`);
          break;
        }
        const items = (resp.json && resp.json.result && resp.json.result.result) || [];
        nextId = resp.json && resp.json.result && resp.json.result.nextId;
        if (!items.length) {
          pushLog(`[${shopEntry.shopName}] 第${page}页无数据，结束翻页`);
          break;
        }
        let newCount = 0;
        for (const it of items) {
          const gid = it.itemGlobalId;
          if (gid !== undefined && gid !== null && seen.has(gid)) continue;
          if (gid !== undefined && gid !== null) seen.add(gid);
          allItems.push(it);
          newCount++;
        }
        pushLog(`[${shopEntry.shopName}] 第${page}页 新增${newCount}条`);
        if (!nextId) break;
        await sleep(200);
      }

      shopEntry.total = allItems.length;
      if (authFailed && !allItems.length) {
        shopEntry.state = 'failed';
        continue;
      }

      pushLog(`[${shopEntry.shopName}] 共${allItems.length}个菜品，开始下载`);
      const manifest = [];
      const usedNames = new Set();
      for (const it of allItems) {
        if (cancelFlag) break;
        const itemName = String(it.itemName || it.itemGlobalId || 'unnamed').trim();
        const photoUrl = it.photoUrl || it.thumbnailUrl;
        if (!photoUrl) {
          shopEntry.skipped++;
          manifest.push({ itemName, itemGlobalId: it.itemGlobalId, photoUrl: null, savedFile: null, status: 'no_photo' });
          continue;
        }
        let finalBase = sanitizeFileName(itemName);
        let n = 1;
        while (usedNames.has(finalBase)) { finalBase = `${sanitizeFileName(itemName)}_${n++}`; }
        usedNames.add(finalBase);
        try {
          const buf = await downloadImage(photoUrl);
          const ext = guessExtFromUrl(photoUrl);
          const fileName = `${finalBase}.${ext}`;
          fs.writeFileSync(path.join(dirInfo.dir, fileName), buf);
          shopEntry.downloaded++;
          manifest.push({ itemName, itemGlobalId: it.itemGlobalId, photoUrl, savedFile: fileName, status: 'downloaded' });
        } catch (e) {
          shopEntry.failed++;
          manifest.push({ itemName, itemGlobalId: it.itemGlobalId, photoUrl, savedFile: null, status: 'download_failed', error: e.message || String(e) });
          pushLog(`[${shopEntry.shopName}] 下载失败 ${itemName}: ${e.message || e}`);
        }
        await sleep(delayMs);
      }

      try {
        fs.writeFileSync(path.join(dirInfo.dir, '_菜品图爬取记录.json'), JSON.stringify(manifest, null, 2), 'utf8');
        if (!shopEntry.failed && !cancelFlag) markShopCompleted(root, shopEntry);
      } catch (e) {
        shopEntry.failed++;
        shopEntry.error = `保存抓取状态失败: ${e.message || e}`;
        pushLog(`[${shopEntry.shopName}] 保存抓取状态失败: ${e.message || e}`);
      }
      shopEntry.state = shopEntry.failed ? 'failed' : cancelFlag ? 'stopped' : 'done';
      pushLog(`[${shopEntry.shopName}] 完成: 成功${shopEntry.downloaded} 跳过${shopEntry.skipped} 失败${shopEntry.failed}`);
      persistState();
    }

    crawlState.status = cancelFlag ? 'stopped' : 'done';
    crawlState.finishedAt = Date.now();
    pushLog('抓取任务结束');
    persistState();
  }

  function stopCrawl(res) {
    if (crawlState.status !== 'running') return sendJson(res, 200, { ok: true, note: '当前没有正在进行的任务' });
    cancelFlag = true;
    pushLog('收到停止请求，将在当前步骤完成后停止');
    persistState();
    sendJson(res, 200, { ok: true });
  }

  return {
    startCrawl,
    stopCrawl,
    getState: () => crawlState,
    getLog: () => crawlState.log,
    clearLog: () => { crawlState.log = []; },
    readCompletionState,
    flushState: persistState,
  };
}

module.exports = { createCrawlService };