hyreal-gen-pic/scripts/eval-ask.js

#!/usr/bin/env node
'use strict';

/**
 * 智能问数 · 金标准评测（口径回归）
 * ----------------------------------------------------------------------------
 * 在隔离数据湖上直跑工具链（sanitize → handler），对真实业务问句断言
 * 「期望工具 + 期望关键数字」。专测口径：加权比率、可比门店集、缺失日、
 * 层间一致性、同行对标、跨主体对比与参数防呆。零网络、零真实模型。
 *
 * 与 smoke 的分工：smoke 管「端点行为没坏」，本脚本管「数字算得对不对」——
 * 改 prompt / 工具 / 口径 / 换模型前后都应跑一遍：node scripts/eval-ask.js
 *
 * 末尾附「真机金标准问句清单」：接真实模型后人工抽查的多轮与表述变体。
 */

const fs = require('fs');
const os = require('os');
const path = require('path');

const ROOT = path.resolve(__dirname, '..');

// ---------- 隔离湖夹具（数字为断言而设计） ----------
// 主体 eval「金标准主体」：一食堂(101)/二食堂(102)@第一餐厅 + 三食堂(103)@第二餐厅
// 本期 20261001-1007（7 天全齐）；上期 0924-0930 中 0925 全体缺数、二食堂 0927 缺数；
// 三食堂仅本期 1004-1007 有数（模拟新开店，测可比集剔除）。
// 关键期望值：compare 可比集(一+二) 10500 vs 7900（+32.9%）；加权进店率 0.0709
// （算术平均会是 0.0567，故意设计两套算法可区分）。

function mkRow(date, scope, ids, m) {
  return {
    date, scope,
    shopId: ids.shopId || '', shopName: ids.shopName || '',
    chainId: ids.chainId || '', chainName: ids.chainName || '',
    bizHours: null, abnormalCloseHours: null, goodsSaleAmt: null,
    syncedAt: new Date().toISOString(),
    validOrderCnt: m.orders || 0, incomeAmt: m.income || 0, grossAmt: m.gross || 0,
    expenseAmt: m.expense || 0, expUv: m.uv || 0, orderUserCnt: m.buyers || 0,
    clkExpRate: m.clk || 0, ordClkRate: 0, perActualPay: 0,
    actOrdCnt: 0, actSubsidyAmt: 0, shopRoi: 0, rebuyRate30d: 0,
    raw: {
      gross_amt: m.gross || 0, valid_order_cnt: m.orders || 0,
      gross_amt_mall_avg: 800,
      gross_amt_mall_avg_rank: m.rank != null ? m.rank : 9,
      valid_order_cnt_diff: m.orders || 0,
    },
  };
}

function buildFixture(lakeDir) {
  const now = new Date().toISOString();
  const rows = [];
  const put = (date, scope, ids, m) => rows.push(mkRow(date, scope, ids, m));
  const A = { shopId: '101', shopName: '一食堂', chainId: '9', chainName: '第一餐厅' };
  const B = { shopId: '102', shopName: '二食堂', chainId: '9', chainName: '第一餐厅' };
  const C = { shopId: '103', shopName: '三食堂', chainId: '8', chainName: '第二餐厅' };
  const curDays = ['20261001', '20261002', '20261003', '20261004', '20261005', '20261006', '20261007'];
  const prevDays = ['20260924', '20260926', '20260927', '20260928', '20260929', '20260930'];
  const Acur = { gross: 1000, income: 800, expense: 200, orders: 100, uv: 2000, clk: 0.10, rank: 1 };
  const Aprev = { gross: 900, income: 720, expense: 180, orders: 90, uv: 1800, clk: 0.10, rank: 1 };
  const Bcur = { gross: 500, income: 400, expense: 100, orders: 40, uv: 1000, clk: 0.02, rank: 4 };
  const Bprev = { gross: 500, income: 400, expense: 100, orders: 40, uv: 1000, clk: 0.02, rank: 4 };
  const Cm = { gross: 300, income: 240, expense: 60, orders: 30, uv: 600, clk: 0.05, rank: 6 };
  for (const d of curDays) {
    put(d, 'shop', A, Acur);
    put(d, 'shop', B, Bcur);
    put(d, 'chain', { chainId: '9', chainName: '第一餐厅' }, { gross: 1500, income: 1200, expense: 300, orders: 140, uv: 3000, clk: 0.08 });
    if (d >= '20261004') {
      put(d, 'shop', C, Cm);
      put(d, 'chain', { chainId: '8', chainName: '第二餐厅' }, { gross: 300, income: 240, expense: 60, orders: 30, uv: 600, buyers: 30, clk: 0.05 });
    }
    const c = { gross: d >= '20261004' ? 1800 : 1500, income: d >= '20261004' ? 1440 : 1200, expense: d >= '20261004' ? 360 : 300, orders: d >= '20261004' ? 170 : 140, uv: d >= '20261004' ? 3600 : 3000, clk: 0.08 };
    put(d, 'company', { chainId: '-1' }, c);
  }
  for (const d of prevDays) {
    put(d, 'shop', A, Aprev);
    if (d !== '20260927') {
      put(d, 'shop', B, Bprev);
      // 连锁/公司行 = 门店合计（一 900 + 二 500）
      put(d, 'chain', { chainId: '9', chainName: '第一餐厅' }, { gross: 1400, income: 1120, expense: 280, orders: 130, uv: 2800, clk: 0.08 });
      put(d, 'company', { chainId: '-1' }, { gross: 1400, income: 1120, expense: 280, orders: 130, uv: 2800, clk: 0.08 });
    } else {
      // 二食堂缺数日：连锁/公司行 = 仅一食堂
      put(d, 'chain', { chainId: '9', chainName: '第一餐厅' }, { gross: 900, income: 720, expense: 180, orders: 90, uv: 1800, clk: 0.10 });
      put(d, 'company', { chainId: '-1' }, { gross: 900, income: 720, expense: 180, orders: 90, uv: 1800, clk: 0.10 });
    }
  }
  fs.writeFileSync(path.join(lakeDir, 'entities.json'), JSON.stringify({ entities: [
    { key: 'eval', firstSeen: now, lastSeen: now, merchantId: 'eval', name: '金标准主体', chainCount: 2, shopCount: 3,
      chains: [{ chainId: '9', chainName: '第一餐厅' }, { chainId: '8', chainName: '第二餐厅' }],
      shops: [
        { shopId: '101', shopName: '一食堂', chainId: '9', chainName: '第一餐厅' },
        { shopId: '102', shopName: '二食堂', chainId: '9', chainName: '第一餐厅' },
        { shopId: '103', shopName: '三食堂', chainId: '8', chainName: '第二餐厅' },
      ] },
    { key: 'evalb', firstSeen: now, lastSeen: now, merchantId: 'evalb', name: '对照主体', chainCount: 1, shopCount: 1,
      chains: [{ chainId: '8', chainName: '第二餐厅' }],
      shops: [{ shopId: '201', shopName: '样本店', chainId: '8', chainName: '第二餐厅' }] },
  ] }));
  fs.mkdirSync(path.join(lakeDir, 'eval'), { recursive: true });
  const byMonth = (list) => JSON.stringify({ version: 2, updated: now, rows: list });
  fs.writeFileSync(path.join(lakeDir, 'eval', 'daily-202609.json'), byMonth(rows.filter((r) => r.date < '20261000')));
  fs.writeFileSync(path.join(lakeDir, 'eval', 'daily-202610.json'), byMonth(rows.filter((r) => r.date >= '20261000')));
  // 异常检测主体：6 天 1000 + 尾日 3000（统计尖峰，LOO z ≈ 6.2）
  fs.mkdirSync(path.join(lakeDir, 'evalc'), { recursive: true });
  const cRows = curDays.map((d) => mkRow(d, 'shop', { shopId: '301', shopName: '爆量店', chainId: '8', chainName: '第二餐厅' }, { gross: d === '20261007' ? 3000 : 1000, income: d === '20261007' ? 2400 : 800, expense: d === '20261007' ? 600 : 200, orders: 100, uv: 2000, clk: 0.08 }));
  fs.writeFileSync(path.join(lakeDir, 'evalc', 'daily-202610.json'), byMonth(cRows));
  // entities.json 追加 evalc
  const entities = JSON.parse(fs.readFileSync(path.join(lakeDir, 'entities.json'), 'utf8'));
  entities.entities.push({ key: 'evalc', firstSeen: now, lastSeen: now, merchantId: 'evalc', name: '异常主体', chainCount: 1, shopCount: 1,
    chains: [{ chainId: '8', chainName: '第二餐厅' }],
    shops: [{ shopId: '301', shopName: '爆量店', chainId: '8', chainName: '第二餐厅' }] });
  fs.writeFileSync(path.join(lakeDir, 'entities.json'), JSON.stringify(entities));
}

// ---------- 评测执行 ----------

const results = [];
function check(name, fn) {
  try { fn(); results.push({ name, ok: true }); }
  catch (e) { results.push({ name, ok: false, error: e.message }); }
}
function expect(cond, msg) { if (!cond) throw new Error(msg); }
function near(a, b, tol) { return Math.abs(a - b) <= (tol || 1e-6); }

function main() {
  const lakeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'genpic-eval-'));
  process.env.GENPIC_LAKE_DIR = lakeDir;
  buildFixture(lakeDir);

  // env 先行，require 在后：确保湖指向隔离目录
  const svc = require(path.join(ROOT, 'napos-service.js'));
  const nlQuery = require(path.join(ROOT, 'nl-query.js'));
  const registry = require(path.join(ROOT, 'server/agent/tools/registry.js'));
  const meta = svc.store.getEntity('eval');
  const ctx = { svc, entityKey: 'eval', meta, shops: meta.shops || [], dateRange: nlQuery.lakeDateRange('eval', svc) };

  const run = (toolName, args) => {
    const tool = registry.get(toolName);
    expect(tool, `工具未注册：${toolName}`);
    return tool.handler(ctx, tool.sanitize(args || {}, ctx));
  };
  const runRaw = (toolName, args) => {
    const tool = registry.get(toolName);
    return tool.sanitize(args || {}, ctx);
  };

  // ---- 查询口径 ----
  check('TopN 排行：一食堂 7000 > 二食堂 3500（二食堂上期缺数不影响本期）', () => {
    const out = run('query_report', { mode: 'shop', from: '20261001', to: '20261007', sort: 'grossAmt', limit: 2 });
    expect(out.data.rows.length === 2, `行数 ${out.data.rows.length}`);
    expect(out.data.rows[0].shopName === '一食堂' && out.data.rows[0].grossAmt === 7000, JSON.stringify(out.data.rows[0]));
    expect(out.data.rows[1].shopName === '二食堂' && out.data.rows[1].grossAmt === 3500, JSON.stringify(out.data.rows[1]));
  });
  check('按日趋势：7 天、缺数日不含三食堂时合计正确', () => {
    const out = run('query_report', { mode: 'date', from: '20261001', to: '20261007', metrics: ['grossAmt', 'validOrderCnt'] });
    expect(out.data.rows.length === 7, `行数 ${out.data.rows.length}`);
    const d3 = out.data.rows.find((r) => r.date === '20261003');
    const d5 = out.data.rows.find((r) => r.date === '20261005');
    expect(d3.grossAmt === 1500 && d5.grossAmt === 1800, `1003=${d3.grossAmt} 1005=${d5.grossAmt}`);
    expect(out.data.summary.grossAmt === 11700 && out.data.summary.validOrderCnt === 1100, JSON.stringify(out.data.summary));
  });
  check('加权比率：进店率按曝光加权 0.0709（算术平均 0.0567 是错的）', () => {
    const out = run('query_report', { mode: 'shop', from: '20261001', to: '20261007', metrics: ['clkExpRate'] });
    const w = out.data.summary.clkExpRate;
    expect(near(w, 0.0709, 1e-6), `加权进店率 ${w}（期望 0.0709）`);
    expect(Math.abs(w - 0.0567) > 0.001, '与算术平均不可区分，加权实现可疑');
  });
  check('门店部分匹配：问「二」只出二食堂', () => {
    const out = run('query_report', { mode: 'shop', from: '20261001', to: '20261007', shops: ['二'] });
    expect(out.data.rows.length === 1 && out.data.rows[0].shopName === '二食堂' && out.data.rows[0].grossAmt === 3500,
      JSON.stringify(out.data.rows));
  });

  // ---- 同比环比口径 ----
  check('环比：可比集剔除新开店，10500 vs 7900（+32.9%）', () => {
    const out = run('compare_periods', {});
    const gross = out.data.deltas.find((d) => d.key === 'grossAmt');
    expect(gross.cur === 10500 && gross.prev === 7900, `环比数字 ${JSON.stringify(gross)}`);
    expect(near(gross.deltaPct, 0.3291, 0.0005), `变化率 ${gross.deltaPct}`);
    expect(out.data.comparableShopCount === 2, `可比门店 ${out.data.comparableShopCount}`);
    expect(out.data.opened.length === 1 && out.data.opened[0].shopName === '三食堂', '新开店未单列');
    expect(out.data.missingDays.length === 0, '本期不应缺日');
  });
  check('环比 verdict：恒等式通过 + 可比集警告（有新开店）', () => {
    const out = run('compare_periods', {});
    const vs = out.data.verdicts || [];
    expect(vs.filter((v) => v.kind === 'identity' && v.status === 'ok').length === 2, '恒等式 verdict 异常');
    const cov = vs.find((v) => v.kind === 'coverage');
    expect(cov && cov.status === 'warn' && /新出现/.test(cov.detail), `coverage verdict 异常：${JSON.stringify(cov)}`);
  });
  check('显式区间环比：等长平移取上期，缺数按实算', () => {
    const out = run('compare_periods', { from: '20261001', to: '20261003' });
    const gross = out.data.deltas.find((d) => d.key === 'grossAmt');
    // 基期 = 0928-0930（三天全齐）：一 2700 + 二 1500 = 4200；本期 = 一 3000 + 二 1500 = 4500
    expect(gross.cur === 4500 && gross.prev === 4200, `环比数字 ${JSON.stringify(gross)}`);
    expect(near(gross.deltaPct, 0.0714, 0.0005), `变化率 ${gross.deltaPct}`);
  });
  check('显式区间环比：上期缺数日按实算不虚构（0925 全体缺数、0927 二食堂缺数）', () => {
    const out = run('compare_periods', { from: '20260926', to: '20260928' });
    const gross = out.data.deltas.find((d) => d.key === 'grossAmt');
    // 本期窗口 0926-0928：1400 + 900 + 1400 = 3700；基期 0923(无)/0924(1400)/0925(无) = 1400
    expect(gross.cur === 3700 && gross.prev === 1400, `环比数字 ${JSON.stringify(gross)}`);
  });

  // ---- 指标层（派生指标与聚合口径） ----
  check('指标层：基础指标逐日序列（按门店行聚合）', () => {
    const out = run('metric_query', { metric: 'grossAmt', from: '20261001', to: '20261003' });
    const v = Object.fromEntries(out.data.series.map((s) => [s.date, s.value]));
    expect(v['20261001'] === 1500 && v['20261003'] === 1500, JSON.stringify(out.data.series));
  });
  check('指标层：7 日移动平均（窗口前置 + 缺数按实算）', () => {
    const out = run('metric_query', { metric: 'gross_mov7', from: '20261001', to: '20261007' });
    const v = Object.fromEntries(out.data.series.map((s) => [s.date, s.value]));
    // 1007：mean(1001..1007) = (1500×3 + 1800×4)/7 ≈ 1671.43
    expect(near(v['20261007'], 11700 / 7, 0.01), `1007 mov7 ${v['20261007']}`);
    // 1001：窗口 0925(缺)/0926(1400)/0927(900)/0928(1400)/0929(1400)/0930(1400)/1001(1500) → 8000/6 ≈ 1333.33
    expect(near(v['20261001'], 8000 / 6, 0.01), `1001 mov7 ${v['20261001']}`);
  });
  check('指标层：加权比率日值（进店率按当日曝光加权）', () => {
    const out = run('metric_query', { metric: 'clkExpRate', from: '20261001', to: '20261001' });
    const v = out.data.series[0].value;
    // 1001：一 (0.10×2000) + 二 (0.02×1000) = 220 / 3000 ≈ 0.0733（三食堂 1004 才开业）
    expect(near(v, 220 / 3000, 0.0001), `加权日值 ${v}`);
  });
  check('指标层：未知指标被拒绝并列出可用清单', () => {
    let rejected = false;
    try { run('metric_query', { metric: 'no_such' }); } catch (e) { rejected = /可用指标/.test(e.message); }
    expect(rejected, '未知指标未拒绝或未给清单');
  });

  // ---- 质量与校验 ----
  check('数据体检：缺 1/14 天为 info、无全零误报', () => {
    const out = run('data_quality', {});
    expect(out.summary.includes('缺 1/14'), `缺日未识别：${out.summary}`);
    expect(out.summary.includes('未发现全零行'), `全零误报：${out.summary}`);
    expect(!out.summary.includes('[fail]'), `健康湖误报 fail：${out.summary}`);
    expect((out.data.verdicts || []).length >= 5, 'verdicts 未产出');
  });
  check('层间核对：13 天 × 双指标全一致', () => {
    const out = run('cross_check', {});
    expect(out.data.checked === 52 && out.data.mismatchCount === 0 && out.data.severe === false,
      `checked=${out.data.checked} mismatch=${out.data.mismatchCount}`);
  });

  // ---- 同行对标与源侧 ----
  check('同行对标：排名升序、自身日均与同行均值正确', () => {
    const out = run('peer_benchmark', { field: 'gross_amt' });
    const shops = out.data.shops;
    expect(shops.length === 3, `门店数 ${shops.length}`);
    expect(shops[0].shopName === '一食堂' && shops[0].rankAvg === 1 && shops[0].peerAvg === 800, JSON.stringify(shops[0]));
    expect(near(shops[0].own, 953.85, 0.01), `自身日均 ${shops[0].own}（期望 (6×900+7×1000)/13=953.85）`);
    expect(shops[2].rankAvg === 6, `末位排名 ${shops[2].rankAvg}`);
  });
  check('源侧字段：raw_inspect 按包含匹配取到 mall 均值与排名', () => {
    const out = run('raw_inspect', { date: '20261001', shops: ['一食堂'], keys: ['gross_amt'] });
    expect(out.data.found && out.data.hasRaw && out.data.matched.length >= 3, JSON.stringify(out.data));
    expect(out.data.values.gross_amt_mall_avg === 800, `均值 ${JSON.stringify(out.data.values)}`);
  });

  // ---- 跨主体与报告 ----
  check('跨主体对比：两主体按本期营业额降序', () => {
    const out = run('cross_entity', {});
    expect(out.data.entities.length === 2, `主体数 ${out.data.entities.length}`);
    expect(out.data.entities[0].name === '金标准主体' && out.data.entities[0].cur.gross === 11700, JSON.stringify(out.data.entities[0]));
    expect(near(out.data.entities[0].grossDeltaPct, 48.1, 0.05), `环比 ${out.data.entities[0].grossDeltaPct}`);
  });
  check('分析报告：URL 组装且区间被钳制', () => {
    const out = run('build_report', { from: '20200101', to: '20991231' });
    expect(out.data.url.includes('entity=eval') && out.data.url.includes('from=20260924') && out.data.url.includes('to=20261007'),
      out.data.url);
  });
  check('分析报告：deep 旗切换到深度报告端点', () => {
    const out = run('build_report', { deep: true });
    expect(out.data.url.includes('/api/agent/report-deep'), out.data.url);
  });

  // ---- 统计异常检测与 LMDI 归因 ----
  check('异常检测：尖峰日 LOO z 判 fail，正常日不误报', () => {
    const ctxC = { svc, entityKey: 'evalc', meta: svc.store.getEntity('evalc'), shops: svc.store.getEntity('evalc').shops || [], dateRange: nlQuery.lakeDateRange('evalc', svc) };
    const out = registry.get('detect_anomalies').handler(ctxC, registry.get('detect_anomalies').sanitize({ metric: 'grossAmt', from: '20261001', to: '20261007' }, ctxC));
    expect(out.data.anomalies.length === 1 && out.data.anomalies[0].date === '20261007', JSON.stringify(out.data.anomalies));
    // LOO 基线 = 其余 6 天（均值 1000、σ 下限 100）→ z = 2000/100 = 20
    expect(out.data.anomalies[0].z >= 10, `z ${out.data.anomalies[0].z}`);
    expect(out.data.verdicts[0].status === 'fail' && out.data.verdicts[0].kind === 'anomaly', JSON.stringify(out.data.verdicts));
  });
  check('异常检测：健康序列不误报', () => {
    const out = run('detect_anomalies', { metric: 'grossAmt', from: '20261001', to: '20261007' });
    expect(out.data.anomalies.length === 0 && out.data.verdicts[0].status === 'ok', JSON.stringify(out.data.anomalies));
  });
  check('LMDI 归因：单店且仅曝光变化时 曝光效应=全部变化（残差≈0）', () => {
    const out = run('attribution', { from: '20261001', to: '20261007', shops: ['一食堂'] });
    const uv = out.data.factors.find((f) => f.key === 'expUv');
    const clk = out.data.factors.find((f) => f.key === 'clkExpRate');
    expect(near(uv.contribution, 1600, 0.5), `曝光效应 ${uv.contribution}`);
    expect(Math.abs(clk.contribution) < 0.01, `进店率效应应为 0：${clk.contribution}`);
    expect(Math.abs(out.data.residual) < 1, `残差 ${out.data.residual}`);
  });
  check('LMDI 归因：多店聚合输出混合残差说明', () => {
    const out = run('attribution', {});
    expect(out.data.residual != null && out.data.factors.length === 3, JSON.stringify(out.data.factors));
  });

  // ---- 参数防呆 ----
  check('防呆：不存在的门店/非法字段/缺日期必须拒绝', () => {
    for (const [tool, args] of [
      ['compare_periods', { shops: ['不存在的店'] }],
      ['peer_benchmark', { field: 'no_such_field' }],
      ['raw_inspect', {}],
    ]) {
      let rejected = false;
      try { runRaw(tool, args); } catch { rejected = true; }
      expect(rejected, `${tool} 未拒绝非法参数：${JSON.stringify(args)}`);
    }
  });

  // ---- 报告 ----
  const failed = results.filter((r) => !r.ok);
  console.log('\n========== 问数金标准评测 ==========');
  for (const r of results) console.log(`${r.ok ? '✓' : '✗'} ${r.name}${r.ok ? '' : '  ← ' + r.error}`);
  console.log(`\n通过 ${results.length - failed.length}/${results.length}`);

  console.log('\n—— 真机金标准问句清单（接真实模型后人工抽查，覆盖表述变体与多轮） ——');
  console.log('1. 上月营业额Top5门店 → 查询排行，数字与湖一致');
  console.log('2. 最近7天营业额和单量趋势 → 按日折线，7 个点');
  console.log('3. 本期比上期营业额变化多少？主要受哪些门店影响 → 可比集口径 + 主因店');
  console.log('4. 二食堂最近进店转化率怎么样 → 加权比率（对照表格页数字）');
  console.log('5. 那一食堂呢（追问） → 上下文门店切换');
  console.log('6. 做一次数据体检，看看有没有问题 → 分级问题清单');
  console.log('7. 哪家店同行对标掉队了 → peer_benchmark + 排名变化');
  console.log('8. 几个食堂里哪家最近表现更好 → cross_entity');
  console.log('9. 云端和本地的数据一致吗 → aitable_verify');
  console.log('10. 给我出一份上周的经营报告 → build_report 链接可打开');

  try { fs.rmSync(lakeDir, { recursive: true, force: true }); } catch { /* ignore */ }
  if (failed.length) process.exit(1);
}

main();