hyreal-gen-pic/napos/classify.js

'use strict';
/**
 * 把抓包流水（traffic.jsonl）归纳成接口目录（apis.json）。
 *
 * 淘宝闪购后台有个特点：**大量调用走通用 RPC 网关**，真实方法名不在 URL 路径里，
 * 而在 query 的 method= 或请求体的 service/method 字段里。例如：
 *
 *   POST /nevermore.goods/invoke?method=FoodService2.searchRecycleBinView
 *        body = { service:'FoodService2', method:'searchRecycleBinView', params:{...} }
 *   POST /xtop/xtop.vas.gateway.iOneRoute.callFromNapos/1.0?service=shopService&method=queryShopAuthority
 *        body = { metas:{...}, params:{ from,to,service,method,params:[...],reqId } }
 *
 * 不把这些拆开，122 个接口会被压成几十个无意义的同名 URL。
 * 本模块负责拆开、抽参数、推结构、判读写。
 */

// ------------------------------------------------------------------ 噪声识别

const NOISE_HOST_SUFFIX = [
  '.alicdn.com', '.elemecdn.com', '.aliapp.org', '.alipay.com', '.1688.com',
  '.koubei.com', '.alibaba-inc.com', '.taobao.com', '.mmstat.com',
  '.oss-accelerate.aliyuncs.com', '.oss-cn-zhangjiakou.aliyuncs.com',
];
/** 例外：同后缀里其实是有用数据的 */
const NOISE_HOST_EXCEPT = [
  'lsycm.alibaba.com',          // 生意参谋/经营数据
  'xux-web-config.oss-accelerate.aliyuncs.com', // 前端配置
];

const NOISE_PATH = [
  /\/arms\.\d/, /\/beacon/, /\/report\b/, /\/track\b/, /\.gif$/, /\.png$/, /\.jpg$/,
  /\/mtb\//, /\/ynuf/, /\/mdap/, /\/druid/, /\.js$/, /\.css$/, /\.woff2?$/,
];

function isNoise(url) {
  let u;
  try { u = new URL(url); } catch { return true; }
  const host = u.hostname;
  const excepted = NOISE_HOST_EXCEPT.some((h) => host === h || host.endsWith('.' + h));
  if (!excepted) {
    const base = NOISE_HOST_SUFFIX.find((s) => host.endsWith(s));
    // .taobao.com 下的 gw/h5api 是真业务网关（agw.alsc.taobao.com）
    if (base === '.taobao.com') {
      if (!/\/gw\//.test(u.pathname)) return true;
    } else if (base) return true;
  }
  return NOISE_PATH.some((r) => r.test(u.pathname));
}

// ------------------------------------------------------------------ 路径模板

function templatePath(pathname) {
  return pathname
    .split('/')
    .map((seg) => {
      if (!seg) return seg;
      if (/^\d{4}-\d{2}-\d{2}/.test(seg)) return '{date}';
      if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(seg)) return '{uuid}';
      if (/^[0-9a-f]{16,}$/i.test(seg)) return '{hash}';
      if (/^\d{4,}$/.test(seg)) return '{id}';
      return seg;
    })
    .join('/');
}

// ------------------------------------------------------------------ 类型草图

function typeOf(v) {
  if (v === null) return 'null';
  if (Array.isArray(v)) return 'array';
  return typeof v;
}

function shape(v, depth = 0) {
  if (depth >= 4) return typeOf(v);
  if (v === null || v === undefined) return 'null';
  if (Array.isArray(v)) return v.length ? [shape(v[0], depth + 1)] : [];
  if (typeof v === 'object') {
    const o = {};
    for (const k of Object.keys(v).slice(0, 40)) o[k] = shape(v[k], depth + 1);
    return o;
  }
  return typeof v;
}

function tryParseJson(s) {
  if (typeof s !== 'string') return undefined;
  const t = s.trim();
  if (!t || (t[0] !== '{' && t[0] !== '[')) return undefined;
  try { return JSON.parse(t); } catch { return undefined; }
}

/** 收集对象里所有叶子字段路径（深度受限），用于"这个接口要传什么" */
function leafKeys(v, prefix = '', out = [], depth = 0) {
  if (depth > 3 || v === null || v === undefined) return out;
  if (Array.isArray(v)) {
    if (v.length) leafKeys(v[0], prefix + '[]', out, depth + 1);
    return out;
  }
  if (typeof v === 'object') {
    for (const k of Object.keys(v)) leafKeys(v[k], prefix ? `${prefix}.${k}` : k, out, depth + 1);
    return out;
  }
  if (prefix) out.push(prefix);
  return out;
}

// ------------------------------------------------------------------ 读写判定

// 注意：这两张词表被 guard.js 复用，改这里等于改写操作防护策略。
//
// 用**词元精确匹配**而不是正则子串匹配，因为子串会大面积撞词：
//   marketingActivity… 里的 mark → 误判成"标记"
//   shopSettings…      里的 set  → 误判成"设置"
//   getTaskCompleteRate 里的 complete → 误判成"完成操作"
// 切成 camelCase 词元后，"marketing" ≠ "mark"、"settings" ≠ "set"、"completeRate" 里的
// "Complete" 虽然等于 "complete"，但它前面已经先命中了 "get"，按**先出现者为准**判为读。
//
// 为什么"先出现者为准"：这类接口的动作动词几乎都在方法名最前面
// （getXxx / queryXxx / createXxx / saveXxx…），后面的多是被操作的业务名词。
const READ_WORDS = new Set([
  'get', 'query', 'list', 'find', 'search', 'page', 'count', 'stat', 'stats', 'statistics',
  'is', 'has', 'exists', 'exist', 'check', 'valid', 'detail', 'details', 'info',
  'view', 'overview', 'gray', 'tree', 'fetch', 'load', 'read', 'preview',
  'export', 'download', 'calc', 'calculate', 'estimate', 'summary', 'insight',
  'trend', 'rank', 'trace', 'config', 'report', 'analysis', 'analyze', 'analyse',
  'dashboard', 'filter',
]);
// 刻意不收 'permission' / 'authority' / 'total'：
//   userPermissionChange 这类名字里，动作词在后面，把名词当读词会导致**漏拦写操作**。
//   而带这些名词的读接口（getBrandVersionPermission、queryShopAuthority）本来就有 get/query 打头。

const WRITE_WORDS = new Set([
  'create', 'update', 'delete', 'remove', 'save', 'submit', 'set', 'modify', 'add', 'edit',
  'cancel', 'confirm', 'publish', 'unbind', 'bind', 'enable', 'disable', 'close', 'open',
  'import', 'upload', 'agree', 'reject', 'refund', 'audit', 'apply', 'batch',
  'start', 'stop', 'push', 'send', 'reply', 'mark', 'offline', 'online',
  'transfer', 'revoke', 'grant', 'incr', 'decr', 'deduct', 'pay', 'sign',
  'clear', 'reset', 'finish', 'withdraw', 'freeze', 'sync', 'move', 'sort', 'arrange',
  'copy', 'clone', 'invalid', 'expire', 'lock', 'unlock', 'change', 'switch', 'join', 'quit',
]);

/**
 * 把标识符切成小写词元。
 *   saveShopInfo            → ['save','shop','info']
 *   marketingActivityEffect → ['marketing','activity','effect']
 *   IGwShopSummary          → ['i','gw','shop','summary']
 *   mtop….user.set          → [...,'user','set']
 *   h5api                   → ['h','5api']
 */
function wordsOf(text) {
  const out = [];
  for (const part of String(text || '').split(/[._\-\/\\\s]+/)) {
    if (!part) continue;
    for (const w of part.split(/(?=[A-Z])|(?<=[a-z])(?=\d)/)) {
      if (w) out.push(w.toLowerCase());
    }
  }
  return out;
}

/** 按"先出现者为准"判定；读词与写词都有可能命中时，取位置靠前的那个 */
function detectKind(text) {
  for (const w of wordsOf(text)) {
    if (READ_WORDS.has(w)) return { kind: 'read', word: w };
    if (WRITE_WORDS.has(w)) return { kind: 'write', word: w };
  }
  return null;
}

/** 取方法名最后一段，例如 ShopSummaryQueryFacade.queryChainSettleListByPage → queryChainSettleListByPage */
function lastSegment(name) {
  if (!name) return '';
  const parts = String(name).split('.');
  return parts[parts.length - 1];
}

/**
 * 从 URL 路径里挖方法名。
 *
 * 大量接口的方法名直接写在路径里，形如：
 *   /xtop/xtop.napos.keeper.shop.queryShopTree/1.0
 *   /gw/h5api/alsc-ad-account.IGwShopSummaryCouponService.offlineExistsCouponActivity
 * 最后一段往往是版本号（1.0），所以不能只看 path 末尾。
 */
function methodFromPath(pathTemplate) {
  for (const s of String(pathTemplate || '').split('/').filter(Boolean)) {
    if (!s.includes('.')) continue;
    if (/^\d+(\.\d+)*$/.test(s)) continue;          // 版本号段 1.0 / 2.1
    const cand = s.split('.').pop();
    if (cand) return cand;
  }
  return '';
}

/**
 * 判定接口是读还是写。
 * @returns {{kind:'read'|'write'|'unknown', source:string}}
 *   unknown = 拿不到任何动作词且不是 GET，无法确认 → 由防护层保守拦截
 */
function inferKind(httpMethod, apiMethod, pathTemplate) {
  // 1) body / query 里的真实方法名，最可信
  if (apiMethod) {
    const d = detectKind(lastSegment(apiMethod));
    if (d) return { kind: d.kind, source: 'name' };
  }
  // 2) 路径里自带的方法名（xtop / gw 风格）
  const pm = methodFromPath(pathTemplate);
  if (pm) {
    const d = detectKind(pm);
    if (d) return { kind: d.kind, source: 'pathMethod' };
  }
  // 3) 整个路径的词元（REST 纯名词路径，如 /api/customer/chain/general/chainPrvtOverview）
  const d = detectKind(pathTemplate);
  if (d) return { kind: d.kind, source: 'path' };

  return { kind: httpMethod === 'GET' ? 'read' : 'unknown', source: 'http' };
}

// ------------------------------------------------------------------ 概述

const GENERIC_INVOKER = /\/invoke\/?$|invoke\/?$|iOneRoute|h5api|callFromNapos|nevermore|xtop\//i;

function buildCatalog(records) {
  const map = new Map();
  const hosts = {};

  for (const r of records) {
    if (!r || !r.url) continue;
    let u;
    try { u = new URL(r.url); } catch { continue; }
    if (!/^https?:/.test(u.protocol)) continue;

    const host = u.hostname;
    hosts[host] = (hosts[host] || 0) + 1;

    const pt = templatePath(u.pathname);
    const body = tryParseJson(r.postData);

    // ---- 拆出真实 RPC 方法名
    let apiService = '';
    let apiMethod = '';
    if (body && typeof body === 'object' && !Array.isArray(body)) {
      if (typeof body.service === 'string') apiService = body.service;
      if (typeof body.method === 'string') apiMethod = body.method;
      // iOneRoute：真名在 body.params 里
      const inner = body.params && !Array.isArray(body.params) ? body.params : null;
      if (inner && typeof inner.method === 'string') {
        apiMethod = inner.method;
        if (typeof inner.service === 'string' && !apiService) apiService = inner.service;
      }
    }
    if (!apiMethod) {
      const qm = u.searchParams.get('method');
      if (qm) {
        if (qm.includes('.') && !apiService) {
          const i = qm.lastIndexOf('.');
          apiService = qm.slice(0, i);
          apiMethod = qm.slice(i + 1);
        } else apiMethod = qm;
      }
    }
    if (!apiService) {
      const qs = u.searchParams.get('service');
      if (qs) apiService = qs;
    }

    const isGeneric = GENERIC_INVOKER.test(u.pathname);
    const qualified = apiMethod ? (apiService ? `${apiService}.${apiMethod}` : apiMethod) : '';
    // 只有通用网关才把方法名并进 key；普通 REST 路径本身就是唯一标识
    const key = qualified && isGeneric
      ? `${r.method || 'GET'} ${host} ${pt} :: ${qualified}`
      : `${r.method || 'GET'} ${host} ${pt}`;

    let e = map.get(key);
    if (!e) {
      e = {
        key,
        method: r.method || 'GET',
        host,
        pathTemplate: pt,
        scheme: u.protocol.replace(':', ''),
        genericInvoker: isGeneric || undefined,
        service: apiService || undefined,
        rpcMethod: apiMethod || undefined,
        count: 0,
        statuses: {},
        queryKeys: [],
        bodyKeys: [],
        paramKeys: [],
        paramShape: undefined,
        shopIds: [],
        mimeTypes: [],
        pages: [],
        noise: isNoise(r.url),
        sampleUrl: r.url,
        // 请求体样本（截断保存）：写操作防护需要它来提取通用网关的真实方法名
        sampleBody: r.postData ? String(r.postData).slice(0, 2000) : undefined,
      };
      map.set(key, e);
    }

    e.count++;
    if (r.status != null) e.statuses[r.status] = (e.statuses[r.status] || 0) + 1;
    if (r.mimeType && !e.mimeTypes.includes(r.mimeType)) e.mimeTypes.push(r.mimeType);
    if (r.page && !e.pages.includes(r.page)) e.pages.push(r.page);

    for (const k of u.searchParams.keys()) {
      if (k === '_' || k === 't') continue;
      if (!e.queryKeys.includes(k)) e.queryKeys.push(k);
    }

    if (body && typeof body === 'object' && !Array.isArray(body)) {
      for (const k of Object.keys(body)) if (!e.bodyKeys.includes(k)) e.bodyKeys.push(k);
      const p = body.params;
      if (p !== undefined) {
        if (!e.paramShape) e.paramShape = shape(p);
        for (const leaf of leafKeys(p)) {
          if (!e.paramKeys.includes(leaf) && e.paramKeys.length < 40) e.paramKeys.push(leaf);
        }
      }
      for (const sid of [body.metas && body.metas.shopId, body.shopId].filter((x) => x != null)) {
        if (!e.shopIds.includes(sid)) e.shopIds.push(sid);
      }
    } else if (r.postData && !e.bodyKeys.length) {
      e.bodyKeys.push('<form-or-raw>');
    }

    if (!e.responseShape || e.responseShape === undefined) {
      if (r.body) {
        const bj = tryParseJson(r.body);
        e.responseShape = bj !== undefined ? shape(bj) : `<${r.mimeType || 'raw'} ${String(r.body).length}B>`;
        e.responseSample = String(r.body).slice(0, 1500);
      }
    }
  }

  const endpoints = [...map.values()];
  for (const e of endpoints) {
    const k = inferKind(e.method, e.rpcMethod, e.pathTemplate);
    e.kind = k.kind;
    e.kindSource = k.source;
    e.statuses = Object.keys(e.statuses).map(Number).sort();
    e.business = !e.noise;
  }

  endpoints.sort((a, b) => {
    if (a.business !== b.business) return a.business ? -1 : 1;
    if (a.host !== b.host) return a.host.localeCompare(b.host);
    return b.count - a.count;
  });

  const biz = endpoints.filter((e) => e.business);

  // 按业务域归组，方便看"这个后台有哪些能力"
  const groups = {};
  const GROUP_RULES = [
    [/order|weborder|fulfill|delivery|trace|afterSale/i, '订单履约'],
    [/goods|item|FoodService|menu|MenuService|spu|sku|deco/i, '商品/门店装修'],
    [/shop|keeper|unit|chain|brand|ShopService|ShopSetting|authority/i, '店铺/账号'],
    [/marketing|activity|coupon|promotion|ad-|advert|balance|bill|fin|settle|account/i, '营销/财务'],
    [/stat|data|report|dashboard|analy|ycm|biz|metric/i, '数据/报表'],
    [/customer|user|member|comment|review|rating/i, '顾客/评价'],
    [/gray|config|permission|isInGray|framework/i, '灰度/配置'],
  ];
  for (const e of biz) {
    const hay = `${e.host} ${e.pathTemplate} ${e.rpcMethod || ''} ${e.service || ''}`;
    let g = '其他';
    for (const [re, name] of GROUP_RULES) { if (re.test(hay)) { g = name; break; } }
    (groups[g] = groups[g] || []).push(e.key);
  }

  return {
    generatedAt: new Date().toISOString(),
    totals: {
      records: records.length,
      endpoints: endpoints.length,
      businessEndpoints: biz.length,
      read: biz.filter((e) => e.kind === 'read').length,
      write: biz.filter((e) => e.kind === 'write').length,
      unknown: biz.filter((e) => e.kind === 'unknown').length,
      hosts: Object.keys(hosts).length,
    },
    hosts,
    groups,
    endpoints,
  };
}

module.exports = {
  buildCatalog, templatePath, shape, isNoise, tryParseJson, leafKeys,
  inferKind, methodFromPath, lastSegment, wordsOf, detectKind, READ_WORDS, WRITE_WORDS,
};