hyreal-gen-pic/napos/harvest.js

'use strict';
/**
 * 定向流量采集（harvest）
 * ============================================================================
 * 背景：napos/data/apis.json 是当年在客户端里抓 5 分钟流量归纳的，页面覆盖不全
 * （订单页只抓到实时接口、没人点过「历史订单」）。本模块补上「定向补抓」能力：
 * 同事在客户端里操作目标页面（翻历史订单、看营业时间设置），本模块通过 CDP
 * Network 域把这段时间内所有业务请求录下来，输出「接口清单」（method/路径/
 * 请求体采样/状态码），供开发侧定位目录外接口。
 *
 * 边界（与运营套件手册一致）：
 *   · 只录不改 —— 不重放、不模拟点击、不读响应体、不碰任何写接口；
 *   · 只录业务域 —— CDN 静态资源/埋点/风控探针一律丢弃。
 */

const cdp = require('./cdp');

/** 只关心这些业务域（后缀匹配） */
const BIZ_HOST_SUFFIXES = ['ele.me', 'elemecdn.com', 'alibaba.com', 'taobao.com', 'koubei.com'];
/** 静态资源/埋点/风控探针 —— 即使在业务域下也丢 */
const NOISE_HOST_RE = /(^|\.)g\.alicdn\.com$|(^|\.)img\.alicdn\.com$|(^|\.)assets\.elemecdn\.com$|(^|\.)alidt\.alicdn\.com$|mdap\.alipay\.com|ynuf\.aliapp\.org|(^|\.)render\.alipay\.com$|(^|\.)gw\.alipayobjects\.com$|xux-web-config\.oss/i;
const STATIC_EXT_RE = /\.(js|css|png|jpe?g|gif|webp|svg|ico|woff2?|ttf|eot|mp4|map|html?)(\?|$)/i;

const state = { running: false, startedAt: null, seconds: 0, stopRequested: false };
let lastResult = null;

function hostAllowed(hostname) {
  const h = String(hostname || '').toLowerCase();
  if (NOISE_HOST_RE.test(h)) return false;
  return BIZ_HOST_SUFFIXES.some((s) => h === s || h.endsWith('.' + s));
}

function status() {
  return {
    running: state.running,
    startedAt: state.startedAt,
    seconds: state.seconds,
    stopRequested: state.stopRequested,
  };
}

/** 请求体/URL 里常带登录态（ksid/secretkey）：落清单前一律打码，报告外发不泄露凭证 */
function redact(s) {
  return String(s).replace(/(ksid|secretkey)(["']?\s*[:=]\s*"?)([^"'&,;\s}\]]+)/gi, '$1$2***');
}

async function runHarvest({ port = cdp.DEFAULT_PORT, seconds = 120 } = {}) {
  if (state.running) {
    const e = new Error('已有定向采集在进行中');
    e.code = 'HARVEST_BUSY';
    throw e;
  }
  const p = await cdp.probe(port);
  if (!p.ok) {
    const e = new Error(`客户端不在线（调试端口 ${port} 不可用）`);
    e.code = 'HARVEST_NO_PORT';
    throw e;
  }
  const secs = Math.max(10, Math.min(600, Number(seconds) || 120));
  state.running = true;
  state.stopRequested = false;
  state.startedAt = new Date().toISOString();
  state.seconds = secs;
  lastResult = null; // 新一轮先清旧结果：失败/中断时绝不把上一次的清单当本次的

  const map = new Map(); // key -> endpoint 条目（statuses 用 Set，最后转数组）
  const sessions = [];

  const record = (pa) => {
    try {
      const req = pa && pa.request;
      if (!req || !/^https?:/i.test(req.url)) return;
      let u;
      try { u = new URL(req.url); } catch { return; }
      if (!hostAllowed(u.hostname)) return;
      if (STATIC_EXT_RE.test(u.pathname)) return;
      const method = String(req.method || 'GET').toUpperCase();
      const key = `${method} ${u.host}${u.pathname}`;
      let e = map.get(key);
      if (!e) {
        e = { key, method, host: u.host, path: u.pathname, count: 0,
          sampleUrl: redact(req.url).slice(0, 800), sampleBody: '', _statuses: new Set() };
        map.set(key, e);
      }
      e.count += 1;
      if (!e.sampleBody && req.postData) e.sampleBody = redact(String(req.postData)).slice(0, 2000);
      return key;
    } catch { return null; }
  };

  try {
    // 逐 target 建会话；单个 target 失败不阻塞（iframe 进程可能拒绝附加）
    const targets = await cdp.listTargets(port).catch(() => []);
    for (const t of targets) {
      if (!t.webSocketDebuggerUrl || !['page', 'iframe', 'webview'].includes(t.type)) continue;
      try {
        const c = new cdp.CDP(t.webSocketDebuggerUrl);
        await c.open(8000);
        const sid = await c.attachTo(t.id);
        const ridKey = new Map(); // requestId -> endpoint key
        c.on('Network.requestWillBeSent', (pa) => {
          const key = record(pa);
          if (key && pa.requestId) ridKey.set(pa.requestId, key);
        });
        c.on('Network.responseReceived', (pa) => {
          try {
            const key = pa && pa.requestId && ridKey.get(pa.requestId);
            const e = key && map.get(key);
            const st = pa && pa.response && pa.response.status;
            if (e && st) e._statuses.add(st);
          } catch { /* ignore */ }
        });
        await c.send('Network.enable', {}, sid);
        sessions.push({ c, sid });
      } catch { /* ignore this target */ }
    }

    const startedAt = Date.now();
    const deadline = startedAt + secs * 1000;
    while (Date.now() < deadline && !state.stopRequested) await cdp.sleep(500);
    const actualSeconds = Math.round((Date.now() - startedAt) / 1000);

    const endpoints = [...map.values()]
      .map(({ _statuses, ...e }) => ({ ...e, statuses: [..._statuses].sort((a, b) => a - b) }))
      .sort((a, b) => b.count - a.count);

    const result = {
      harvestedAt: new Date().toISOString(),
      seconds: actualSeconds,
      stoppedEarly: state.stopRequested,
      targetCount: targets.length,
      sessionCount: sessions.length,
      endpoints,
    };
    lastResult = result;
    return result;
  } finally {
    // 无论正常结束、提前停止还是半路抛异常：会话必关、状态必清，绝不卡死下次采集
    for (const s of sessions) {
      try { await s.c.detach(s.sid); } catch { /* ignore */ }
      s.c.close();
    }
    state.running = false;
    state.stopRequested = false;
  }
}

/** 提前收工：把剩余等待截断，正常走收尾（会话已录到的请求不丢） */
function stopHarvest() {
  if (!state.running) return status();
  state.stopRequested = true;
  return status();
}

function getResult() { return lastResult; }

module.exports = { runHarvest, stopHarvest, status, getResult, hostAllowed };