Explorar el Código

fix: 云端推送口径与拦截判据修正(拉取不算待推、只读机恒0、层间矛盾判据、错误徽标保留)+ smoke 主服务隔离(GENPIC_CONFIG)

KenZ1117 hace 2 semanas
padre
commit
3014167eb1
Se han modificado 6 ficheros con 78 adiciones y 26 borrados
  1. 6 2
      public/app.js
  2. 2 1
      public/index.html
  3. 8 5
      public/reports.js
  4. 21 1
      scripts/smoke.js
  5. 4 1
      server/context.js
  6. 37 16
      server/services/aitable.js

+ 6 - 2
public/app.js

@@ -268,8 +268,12 @@
         return data;
       }
       if (kind === 'push') {
-        const stale = data.skippedStale ? `,跳过云端较新 ${data.skippedStale} 行` : '';
-        if (status) { status.textContent = `已推送 ${data.pushed} 行(新增 ${data.created} / 更新 ${data.updated}${stale})`; status.className = 'badge badge-ok'; }
+        if (data.pushed === 0) {
+          if (status) { status.textContent = '本地与云端一致,无需推送'; status.className = 'badge badge-ok'; }
+        } else {
+          const stale = data.skippedStale ? `,跳过云端较新 ${data.skippedStale} 行` : '';
+          if (status) { status.textContent = `已推送 ${data.pushed} 行(新增 ${data.created} / 更新 ${data.updated}${stale})`; status.className = 'badge badge-ok'; }
+        }
       } else {
         const detail = (data.accounts || []).map((a) => `${a.account} ${a.rows}行`).join('、') || '无数据';
         if (status) { status.textContent = `已拉取 ${data.pulled} 行:${detail}`; status.className = 'badge badge-ok'; }

+ 2 - 1
public/index.html

@@ -374,7 +374,8 @@
                   <ol class="help-popover-steps">
                     <li><b>不产生重复:</b>每一行的身份是「日期×账号×维度×门店」,多台电脑写同一行只会覆盖,不会重复。</li>
                     <li><b>数据新者胜:</b>推送时对比本地取数时间与云端修改时间,旧数据不会覆盖云端新数据(跳过并计数)。</li>
-                    <li><b>全零拦截:</b>大批量指标全零(疑似取数异常)时拒绝推送,防止坏数据污染云端主档案。</li>
+                    <li><b>层间矛盾拦截:</b>门店明细全部全零而连锁/公司汇总有数(取数异常形态)时拒绝推送;新店多导致的大面积全零是正常形态,不受影响。</li>
+                    <li><b>拉取不算待推:</b>云端拉取回来的数据本来就已在云端,不计入本机「待推送」;从未本地同步过的只读机器恒显示与云端一致。</li>
                     <li><b>拉取以云端为准:</b>其他电脑执行「云端拉取」后,本地同键行被云端版本覆盖。</li>
                   </ol>
                 </div>

+ 8 - 5
public/reports.js

@@ -1551,20 +1551,22 @@
 
   // ---------- 初始化 ----------
 
-  // 数据采集页工具栏:推送当前主体到云端(本地湖 → 钉钉 AI 表格)
+  // 数据采集页工具栏:推送当前主体到云端(本地湖 → 钉钉 AI 表格)。返回是否推送成功。
   async function pushCurrentEntity() {
     const badge = $('reportPushBadge');
     const entity = state.entity || (state.entities[0] || {}).key;
     if (!entity) {
       badge.textContent = '暂无主体';
       badge.className = 'badge badge-fail';
-      return;
+      return false;
     }
     if (window.shanAitableAction) {
       try {
         await window.shanAitableAction('push', 'reportPushBadge', { entity });
-      } catch { /* 徽标已由 aitableAction 写入 */ }
+        return true;
+      } catch { return false; /* 徽标已由 aitableAction 写入,保留错误给用户读 */ }
     }
+    return false;
   }
 
   async function refreshPushBadge() {
@@ -1585,7 +1587,7 @@
         badge.title = '「推送云端」会把上次推送之后的全部增量一次性推送至钉钉 AI 表格';
         badge.className = 'badge badge-warn';
       } else {
-        badge.textContent = '已同步至云端';
+        badge.textContent = '与云端一致';
         badge.title = '';
         badge.className = 'badge badge-ok';
       }
@@ -1597,7 +1599,8 @@
       state.inited = true;
       bindEvents();
       $('btnReportPush').addEventListener('click', async () => {
-        await pushCurrentEntity();
+        const ok = await pushCurrentEntity();
+        if (!ok) return; // 失败:保留错误徽标原样(待推计数也没变)
         renderSyncHistory(); // 云端推送状态列随推送结果刷新
         refreshPushBadge();  // 待推计数归零(或显示剩余)
       });

+ 21 - 1
scripts/smoke.js

@@ -10,6 +10,7 @@
 const { spawn, spawnSync } = require('child_process');
 const net = require('net');
 const http = require('http');
+const fs = require('fs');
 const os = require('os');
 const path = require('path');
 
@@ -113,8 +114,14 @@ function enc(s) { return encodeURIComponent(s); }
 
 async function main() {
   const port = await pickPort();
+  // 主服务隔离:临时湖 + 临时配置(aitable 关闭)——smoke 绝不触碰本机数据湖、config.json 与真实云端凭据
+  const tmpMain = fs.mkdtempSync(path.join(os.tmpdir(), 'genpic-smoke-main-'));
+  const tmpMainLake = path.join(tmpMain, 'lake');
+  fs.mkdirSync(tmpMainLake, { recursive: true });
+  const tmpMainConfig = path.join(tmpMain, 'config.json');
+  fs.writeFileSync(tmpMainConfig, JSON.stringify({ aitable: { enabled: false } }));
   const child = spawn(process.execPath, [path.join(ROOT, 'server.js')], {
-    env: { ...process.env, PORT: String(port), HOST: '127.0.0.1' },
+    env: { ...process.env, PORT: String(port), HOST: '127.0.0.1', GENPIC_LAKE_DIR: tmpMainLake, GENPIC_CONFIG: tmpMainConfig },
     stdio: ['ignore', 'pipe', 'pipe'],
   });
   let bootLog = '';
@@ -339,6 +346,19 @@ async function main() {
       expect(sheets.detail === '门店明细' && sheets.chain === '连锁汇总' && sheets.company === '公司汇总' && fingerprint.split('|').length === 3,
         JSON.stringify(sheets));
     });
+    await check('detectLayerConflict → 门店全零且汇总有数才拦;新店型全零/未营业/无门店行放行', async () => {
+      const mk = (scope, extra = {}) => ({ scope, shopId: scope === 'shop' ? 's1' : '', chainId: scope === 'company' ? '-1' : 'c1', validOrderCnt: 0, incomeAmt: 0, ...extra });
+      // 已知 bug 形态:门店行全零、连锁/公司有数 → 拦
+      const bug = [mk('shop'), mk('chain', { validOrderCnt: 138, incomeAmt: 2900 }), mk('company', { validOrderCnt: 138, incomeAmt: 2900 })];
+      expect(aitableLib.detectLayerConflict(bug) && aitableLib.detectLayerConflict(bug).shopRows === 1, '门店全零而汇总有数未检出');
+      // 新店型:大面积全零但部分门店有数 → 放行(>80% 全零也正常)
+      const newShops = [mk('shop'), mk('shop'), mk('shop', { validOrderCnt: 98, incomeAmt: 1400 }), mk('chain', { validOrderCnt: 138 }), mk('company')];
+      expect(aitableLib.detectLayerConflict(newShops) === null, '部分门店有数被误拦');
+      // 全新未营业:门店与汇总都没数 → 放行
+      expect(aitableLib.detectLayerConflict([mk('shop'), mk('chain'), mk('company')]) === null, '未营业主体被误拦');
+      // 无门店行 → 放行
+      expect(aitableLib.detectLayerConflict([mk('chain', { validOrderCnt: 5 })]) === null, '无门店行被误拦');
+    });
 
     // ---- 问数两段式:结果摘要与兜底话术(纯函数,不调对话模型) ----
     const nlQueryLib = require('../nl-query.js');

+ 4 - 1
server/context.js

@@ -9,7 +9,10 @@ const path = require('path');
 
 const ROOT_DIR = path.resolve(__dirname, '..');
 const PUBLIC_DIR = path.join(ROOT_DIR, 'public');
-const CONFIG_PATH = path.join(ROOT_DIR, 'config.json');
+// GENPIC_CONFIG:把配置文件指到别处(smoke/多实例隔离用),避免触碰本机真实 config.json
+const CONFIG_PATH = process.env.GENPIC_CONFIG
+  ? path.resolve(process.env.GENPIC_CONFIG)
+  : path.join(ROOT_DIR, 'config.json');
 const CONFIG_EXAMPLE_PATH = path.join(ROOT_DIR, 'config.example.json');
 const SESSION_PATH = path.join(ROOT_DIR, 'session.json');
 const MANUAL_SHOPS_PATH = path.join(ROOT_DIR, 'manual-shops.json');

+ 37 - 16
server/services/aitable.js

@@ -75,6 +75,19 @@ const DIMENSION_LABEL = { company: '公司', chain: '连锁', shop: '门店', su
 // scope → 目标表配置键(summary 不上云)
 const SCOPE_SHEET_KEY = { shop: 'sheetDetail', chain: 'sheetChain', company: 'sheetCompany' };
 
+/** 层间矛盾检测(纯函数,推送前最后一道闸):
+ *  门店明细行存在且指标全部全零、同时连锁/公司汇总行有非零指标 → 已知门店级取数异常形态。
+ *  新店多导致的大比例全零(部分门店有数)与全新未营业主体(汇总也没数)都不命中。 */
+function detectLayerConflict(rows) {
+  const shopRows = rows.filter((r) => r.scope === 'shop');
+  if (!shopRows.length) return null;
+  if (!shopRows.every((r) => METRIC_FIELDS.every((f) => !Number(r[f])))) return null;
+  const summaryHasData = rows.some((r) => (r.scope === 'chain' || r.scope === 'company')
+    && METRIC_FIELDS.some((f) => Number(r[f]) > 0));
+  if (!summaryHasData) return null;
+  return { shopRows: shopRows.length };
+}
+
 function rowToRecord(row, accountName) {
   const fields = {
     [FIELD_COLUMNS.date]: row.date || '',
@@ -338,25 +351,22 @@ function createAitableService(ctx) {
 
     const state = readPushState();
     const prev = state[key] || {};
-    const watermark = opts.force || prev.sheets !== fingerprint ? 0 : (prev.lastPushedAt || 0);
-    let rows;
-    if (watermark) {
-      const all = naposService.store.loadRows(key) || [];
-      rows = all.filter((r) => new Date(r.syncedAt || 0).getTime() > watermark);
-    } else {
-      rows = naposService.store.loadRows(key) || [];
-    }
+    // 水位取推送/拉取时间的较大者:拉取回来的行云端本来就有,不作为本地增量重推
+    const watermark = opts.force || prev.sheets !== fingerprint
+      ? 0
+      : Math.max(prev.lastPushedAt || 0, prev.lastPullAt || 0);
+    const all = naposService.store.loadRows(key) || [];
+    const rows = watermark ? all.filter((r) => new Date(r.syncedAt || 0).getTime() > watermark) : all;
     const parts = splitRowsBySheet(rows);
     const pushed = parts.detail.length + parts.chain.length + parts.company.length;
     if (!pushed) return { pushed: 0, created: 0, updated: 0, skippedSummary: parts.skippedSummary, skipped: '无新增数据' };
     // 大批量(尤其首推全量)耗时较长,开始时先留痕,避免长时间静默
     logs.pushSystemLog(`推送云端开始:${accountName}(${watermark ? '增量' : '全量'} ${pushed} 行)`, 'info');
 
-    // 全零保护:大批量门店行指标全零基本可断定取数异常(已知门店级 bug 的症状),
-    // 防止把坏数据灌进云端主档案;确认要推(force)可绕过。
-    const zeroRows = parts.detail.filter((r) => METRIC_FIELDS.every((f) => !Number(r[f]))).length;
-    if (!opts.force && parts.detail.length >= 20 && zeroRows / parts.detail.length > 0.8) {
-      throw err(`本批门店行 ${parts.detail.length} 行中 ${zeroRows} 行指标全零,疑似取数异常,已拦截推送(确认无误可在数据中台强制推送)`, 'AITABLE_ALL_ZERO');
+    // 层间矛盾保护:防止把取数异常的坏数据灌进云端主档案;force 可绕过。
+    const conflict = !opts.force && detectLayerConflict(all);
+    if (conflict) {
+      throw err(`门店明细 ${conflict.shopRows} 行指标全部全零,而连锁/公司汇总行有数据——门店级取数异常的已知形态,已拦截推送,请先排查取数或重新同步`, 'AITABLE_ALL_ZERO');
     }
 
     const token = await getToken();
@@ -378,7 +388,7 @@ function createAitableService(ctx) {
     }
 
     const now = Date.now();
-    state[key] = { lastPushedAt: now, pushedRows: pushed, sheets: fingerprint, at: new Date().toISOString() };
+    state[key] = { ...state[key], lastPushedAt: now, pushedRows: pushed, sheets: fingerprint, at: new Date().toISOString() };
     writePushState(state);
     return { pushed, created, updated, skippedStale, skippedSummary: parts.skippedSummary, perSheet, account: accountName };
   }
@@ -414,6 +424,8 @@ function createAitableService(ctx) {
     if (!groups.size) return { pulled: 0, accounts: [], message: '云端没有可拉取的数据' };
 
     const result = { pulled: 0, accounts: [] };
+    const state = readPushState();
+    const pulledKeys = [];
     for (const [acc, rows] of groups) {
       let key = naposService.store.sanitizeKey(acc);
       const meta = naposService.store.getEntity(key);
@@ -422,9 +434,14 @@ function createAitableService(ctx) {
         key = created.entity.key;
       }
       const res = naposService.store.upsertRows(key, rows);
+      pulledKeys.push(key);
       result.pulled += rows.length;
       result.accounts.push({ account: acc, rows: rows.length, added: res.added, updated: res.updated });
     }
+    // 留痕拉取水位:拉回的行云端已有,不计入本机「待推送」(消费方:pendingCount / pushLake 水位)
+    const now = Date.now();
+    for (const k of new Set(pulledKeys)) state[k] = { ...(state[k] || {}), lastPullAt: now };
+    writePushState(state);
     return result;
   }
 
@@ -452,13 +469,16 @@ function createAitableService(ctx) {
     return s;
   }
 
-  /** 待推送行数:水位之后有变化(syncedAt 更新)的行;配置不齐返回 null(调用方显示配置态) */
+  /** 待推送行数:水位(最近推送/最近拉取取较大者)之后有变化的行;配置不齐返回 null(调用方显示配置态)。
+   *  本机从未本地同步过的主体(只读机:数据全部来自云端拉取)没有「待推送」概念,恒 0。 */
   function pendingCount(entity) {
     const cfg = config.aitable || {};
     if (!cfg.enabled || !cfg.baseId || !cfg.appKey || !cfg.operatorId) return null;
     const naposService = require('../../napos-service.js');
     const key = naposService.store.sanitizeKey(entity);
-    const watermark = (readPushState()[key] || {}).lastPushedAt || 0;
+    if (!naposService.store.loadSyncLogs(key).length) return 0;
+    const st = readPushState()[key] || {};
+    const watermark = Math.max(st.lastPushedAt || 0, st.lastPullAt || 0);
     const all = naposService.store.loadRows(key) || [];
     if (!watermark) return all.length;
     return all.filter((r) => new Date(r.syncedAt || 0).getTime() > watermark).length;
@@ -488,4 +508,5 @@ function createAitableService(ctx) {
 module.exports = {
   createAitableService, FIELD_COLUMNS, DIMENSION_LABEL, SHEET_DEFAULTS,
   rowToRecord, recordKey, recordToRow, splitRowsBySheet, resolveSheets, pushSummary,
+  detectLayerConflict,
 };