Răsfoiți Sursa

refactor: slim collection page to sync console; move report browsing + all-fields view to lake console

KenZ1117 2 săptămâni în urmă
părinte
comite
1f2fc60226
5 a modificat fișierele cu 206 adăugiri și 101 ștergeri
  1. 6 3
      public/app.js
  2. 75 54
      public/index.html
  3. 1 38
      public/lake.js
  4. 117 6
      public/reports.js
  5. 7 0
      server/routes/reports.js

+ 6 - 3
public/app.js

@@ -343,14 +343,17 @@
     if (name === 'tasks' && window.shanTasksInit) window.shanTasksInit();
     if (name === 'history' && window.shanHistoryInit) window.shanHistoryInit();
     if (name === 'reports' && window.shanReportsInit) window.shanReportsInit();
-    if (name === 'lake' && window.shanLakeInit) window.shanLakeInit();
+    if (name === 'lake') {
+      if (window.shanReportsInit) window.shanReportsInit(); // 明细浏览(含全部字段)已挪到中台
+      if (window.shanLakeInit) window.shanLakeInit();
+    }
     if (name === 'analysis' && window.shanAnalysisActivate) window.shanAnalysisActivate();
     if (name === 'settings') initSettings();
     // 系统日志:仅日志页激活时轮询
     if (name === 'logs') startLogPolling();
     else stopLogPolling();
-    // 智能问数机器人:仅经营分析相关页面显示
-    if (window.shanSetAnalysisVisible) window.shanSetAnalysisVisible(name === 'reports' || name === 'analysis');
+    // 智能问数机器人:数据采集(问数入口语义在报表浏览,浏览已挪中台但采集页仍是分析组入口)、数据中台、趋势图表显示
+    if (window.shanSetAnalysisVisible) window.shanSetAnalysisVisible(name === 'reports' || name === 'lake' || name === 'analysis');
   }
 
   navItems.forEach((n) => n.addEventListener('click', () => switchView(n.dataset.view)));

+ 75 - 54
public/index.html

@@ -269,7 +269,7 @@
         <div class="page-header">
           <div>
             <h1>数据采集</h1>
-            <div class="sub">同步商家客户端经营数据到本地数据湖,多维报表与导出</div>
+            <div class="sub">同步控制台:连接商家客户端、按区间取数入库、推送云端主档案</div>
           </div>
           <div class="toolbar">
             <span class="conn-help-wrap">
@@ -304,6 +304,73 @@
 
         <div class="entity-tabs" id="entityTabs"></div>
 
+        <div class="panel panel-fixed" id="syncHistoryPanel">
+          <div class="panel-title">
+            <span>同步历史</span>
+            <span class="badge badge-info" id="syncHistoryBadge">本主体最近 8 次</span>
+          </div>
+          <div class="table-wrap">
+            <table class="data-table">
+              <thead><tr><th>时间</th><th>区间</th><th>范围</th><th>新增</th><th>更新</th><th>失败</th><th>备注</th></tr></thead>
+              <tbody id="syncHistoryRows"><tr><td colspan="7" class="empty-cell">暂无同步记录</td></tr></tbody>
+            </table>
+          </div>
+        </div>
+
+
+      </main>
+    </div>
+
+    <!-- ==================== 数据中台 ==================== -->
+    <div id="viewLake" class="view">
+      <main class="content">
+        <div class="page-header">
+          <div>
+            <h1>数据中台</h1>
+            <div class="sub">本地数据湖全景与明细数据(含全部原始字段),可从云端拉取补全</div>
+          </div>
+          <div class="toolbar">
+            <button class="btn btn-ghost btn-small" id="btnLakeRefresh">
+              <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.8" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M20 12a8 8 0 1 1-2.3-5.6"/><path d="M20 4v5h-5"/></svg>
+              刷新
+            </button>
+            <button class="btn btn-ghost btn-small" id="btnLakePull" title="以钉钉 AI 表格(云端主档案)为准,拉回本地湖;未注册的账号自动创建主体">
+              <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.8" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M12 3v12"/><path d="M7 10l5 5 5-5"/><path d="M4 21h16"/></svg>
+              从云端拉取
+            </button>
+            <span class="badge" id="lakeSyncBadge"></span>
+          </div>
+        </div>
+
+        <div class="stats-grid" id="lakeKpis">
+          <div class="metric"><span class="metric-body"><span class="metric-label">数据主体</span><span class="metric-value" id="lakeEntityCount">–</span><span class="metric-sub">已注册的商家账号</span></span></div>
+          <div class="metric"><span class="metric-body"><span class="metric-label">湖内总行数</span><span class="metric-value" id="lakeRowCount">–</span><span class="metric-sub">按 日期×维度×门店 幂等去重</span></span></div>
+          <div class="metric"><span class="metric-body"><span class="metric-label">日期覆盖</span><span class="metric-value" id="lakeCoverage">–</span><span class="metric-sub">最早 ~ 最晚(后台约 180 天)</span></span></div>
+          <div class="metric"><span class="metric-body"><span class="metric-label">最近同步</span><span class="metric-value" id="lakeLastSync">–</span><span class="metric-sub">取数写入本地湖的时间</span></span></div>
+        </div>
+
+        <div class="panel panel-fixed report-table-panel">
+          <div class="panel-title">
+            <span>主体明细</span>
+            <span class="badge badge-info" id="lakeRowsBadge">0 主体</span>
+          </div>
+          <div class="table-wrap">
+            <table class="data-table">
+              <thead>
+                <tr>
+                  <th>账号名</th><th>门店/连锁</th><th>总行数</th><th>公司</th><th>连锁</th><th>门店</th><th>汇总</th><th>覆盖区间</th><th>最近同步</th><th>上次推送云端</th>
+                </tr>
+              </thead>
+              <tbody id="lakeEntityRows">
+                <tr><td colspan="10" class="empty-cell">加载中…</td></tr>
+              </tbody>
+            </table>
+          </div>
+          <div class="help">冲突策略:同一行(日期×账号×维度×门店)多机写入时以取数时间新者为准;云端更新的行不会被本机旧数据覆盖。大批量全零数据会被拦截,防止取数异常污染云端主档案。</div>
+        </div>
+
+        <div class="entity-tabs" id="lakeEntityTabs"></div>
+
         <div class="report-toolbar panel">
           <div class="field report-field-sm">
             <label>日期范围</label>
@@ -344,6 +411,13 @@
             <label>搜索门店</label>
             <input type="text" id="reportSearch" placeholder="输入门店名称过滤">
           </div>
+          <div class="field">
+            <label>字段</label>
+            <div class="seg-control" id="reportFieldModes" role="tablist">
+              <button type="button" class="active" data-fields="report">报表列</button>
+              <button type="button" data-fields="all">全部字段</button>
+            </div>
+          </div>
           <div class="report-toolbar-actions">
             <button class="btn btn-ghost btn-small" id="btnColumnCfg">列设置</button>
             <button class="btn btn-ghost btn-small" id="btnExportCsv">导出 CSV</button>
@@ -368,60 +442,7 @@
             </div>
           </div>
         </div>
-      </main>
-    </div>
 
-    <!-- ==================== 数据中台 ==================== -->
-    <div id="viewLake" class="view">
-      <main class="content">
-        <div class="page-header">
-          <div>
-            <h1>数据中台</h1>
-            <div class="sub">本地数据湖全景:主体覆盖、行数分布、云端同步状态</div>
-          </div>
-          <div class="toolbar">
-            <button class="btn btn-ghost btn-small" id="btnLakeRefresh">
-              <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.8" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M20 12a8 8 0 1 1-2.3-5.6"/><path d="M20 4v5h-5"/></svg>
-              刷新
-            </button>
-            <button class="btn btn-ghost btn-small" id="btnLakePull" title="以钉钉 AI 表格(云端主档案)为准,拉回本地湖;未注册的账号自动创建主体">
-              <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.8" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M12 3v12"/><path d="M7 10l5 5 5-5"/><path d="M4 21h16"/></svg>
-              从云端拉取
-            </button>
-            <button class="btn btn-primary btn-small" id="btnLakePush" title="把本地湖全部主体的新增/变更行推送到钉钉 AI 表格">
-              <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.8" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M12 21V9"/><path d="M7 14l5-5 5 5"/><path d="M4 3h16"/></svg>
-              推送云端
-            </button>
-            <span class="badge" id="lakeSyncBadge"></span>
-          </div>
-        </div>
-
-        <div class="stats-grid" id="lakeKpis">
-          <div class="metric"><span class="metric-body"><span class="metric-label">数据主体</span><span class="metric-value" id="lakeEntityCount">–</span><span class="metric-sub">已注册的商家账号</span></span></div>
-          <div class="metric"><span class="metric-body"><span class="metric-label">湖内总行数</span><span class="metric-value" id="lakeRowCount">–</span><span class="metric-sub">按 日期×维度×门店 幂等去重</span></span></div>
-          <div class="metric"><span class="metric-body"><span class="metric-label">日期覆盖</span><span class="metric-value" id="lakeCoverage">–</span><span class="metric-sub">最早 ~ 最晚(后台约 180 天)</span></span></div>
-          <div class="metric"><span class="metric-body"><span class="metric-label">最近同步</span><span class="metric-value" id="lakeLastSync">–</span><span class="metric-sub">取数写入本地湖的时间</span></span></div>
-        </div>
-
-        <div class="panel panel-fixed report-table-panel">
-          <div class="panel-title">
-            <span>主体明细</span>
-            <span class="badge badge-info" id="lakeRowsBadge">0 主体</span>
-          </div>
-          <div class="table-wrap">
-            <table class="data-table">
-              <thead>
-                <tr>
-                  <th>账号名</th><th>门店/连锁</th><th>总行数</th><th>公司</th><th>连锁</th><th>门店</th><th>汇总</th><th>覆盖区间</th><th>最近同步</th><th>上次推送云端</th>
-                </tr>
-              </thead>
-              <tbody id="lakeEntityRows">
-                <tr><td colspan="10" class="empty-cell">加载中…</td></tr>
-              </tbody>
-            </table>
-          </div>
-          <div class="help">冲突策略:同一行(日期×账号×维度×门店)多机写入时以取数时间新者为准;云端更新的行不会被本机旧数据覆盖。大批量全零数据会被拦截,防止取数异常污染云端主档案。</div>
-        </div>
       </main>
     </div>
 

+ 1 - 38
public/lake.js

@@ -84,43 +84,7 @@
     }
   }
 
-  // 推送:逐主体执行,聚合结果(单主体失败不中断其余,最后统一汇报)
-  async function pushAll() {
-    const btn = $('btnLakePush');
-    btn.disabled = true;
-    setBadge('lakeSyncBadge', '推送中…');
-    try {
-      const res = await fetch('/api/reports/lake');
-      const data = await res.json();
-      const targets = (data.entities || []).map((e) => e.key);
-      if (!targets.length) throw new Error('本地湖为空,没有可推送的主体');
-      let created = 0;
-      let updated = 0;
-      let stale = 0;
-      const errors = [];
-      for (const key of targets) {
-        try {
-          const r = await window.shanAitableAction('push', 'lakeSyncBadge', { entity: key });
-          created += r.created || 0;
-          updated += r.updated || 0;
-          stale += r.skippedStale || 0;
-        } catch (e) {
-          errors.push(`${key}:${e.message}`);
-        }
-      }
-      if (errors.length) {
-        setBadge('lakeSyncBadge', `部分失败(成功 新增${created}/更新${updated}):${errors[0]}`, 'badge-fail');
-      } else {
-        setBadge('lakeSyncBadge', `已推送:新增 ${created} / 更新 ${updated}${stale ? ` / 跳过云端较新 ${stale}` : ''}`, 'badge-ok');
-      }
-      await loadLake();
-    } catch (error) {
-      setBadge('lakeSyncBadge', error.message, 'badge-fail');
-    } finally {
-      btn.disabled = false;
-    }
-  }
-
+  // 推送动作在数据采集页(取数机的动作);中台只负责看数与从云端拉取
   async function pullAll() {
     const btn = $('btnLakePull');
     btn.disabled = true;
@@ -136,7 +100,6 @@
     if (!inited) {
       inited = true;
       $('btnLakeRefresh').addEventListener('click', loadLake);
-      $('btnLakePush').addEventListener('click', pushAll);
       $('btnLakePull').addEventListener('click', pullAll);
     }
     loadLake();

+ 117 - 6
public/reports.js

@@ -90,6 +90,7 @@
     entity: null,
     preset: '30',
     mode: 'detail',
+    rawView: false,
     shopFilter: new Set(),
     q: '',
     columns: [],
@@ -141,7 +142,7 @@
   }
 
   function renderEntityTabs() {
-    for (const containerId of ['entityTabs', 'analysisEntityTabs']) {
+    for (const containerId of ['entityTabs', 'analysisEntityTabs', 'lakeEntityTabs']) {
       const wrap = $(containerId);
       if (!wrap) continue;
       wrap.innerHTML = '';
@@ -280,6 +281,30 @@
   }
 
   function tableColumns() {
+    // 全部字段模式(数据中台):固定业务列 + 从首行 raw 发现的全部原始字段(原始 API 字段名)
+    if (state.rawView) {
+      const base = [
+        { key: 'account', label: '账号名', type: 'text' },
+        { key: 'date', label: '日期', type: 'date' },
+        { key: 'scope', label: '维度', type: 'text' },
+        { key: 'shopId', label: '门店ID', type: 'text' },
+        { key: 'shopName', label: '门店名称', type: 'text' },
+      ];
+      const rawCols = [];
+      const seen = new Set(base.map((c) => c.label));
+      for (const r of state.rows) {
+        const raw = r && r.raw;
+        if (!raw || typeof raw !== 'object') continue;
+        for (const k of Object.keys(raw)) {
+          if (seen.has(k)) continue;
+          seen.add(k);
+          const sample = raw[k];
+          const isNum = typeof sample === 'number' || (typeof sample === 'string' && sample !== '' && Number.isFinite(Number(sample)));
+          rawCols.push({ key: `raw.${k}`, label: k, type: isNum ? 'num' : 'text', raw: true });
+        }
+      }
+      return [...base, ...rawCols];
+    }
     const metricCols = metricColumns();
     const accountCol = { key: 'account', label: '账号名', type: 'text' };
     if (state.mode === 'shop') return [accountCol, { key: 'shopName', label: '门店名称', type: 'text' }, ...metricCols];
@@ -287,9 +312,17 @@
     return [accountCol, { key: 'date', label: '日期', type: 'date' }, { key: 'shopName', label: '门店名称', type: 'text' }, ...metricCols];
   }
 
+  // 取单元格值:raw. 前缀深入原始字段对象;对象/数组序列化展示
+  function cellValue(r, key) {
+    if (!key.startsWith('raw.')) return r[key];
+    const raw = r && r.raw;
+    if (!raw || typeof raw !== 'object') return undefined;
+    return raw[key.slice(4)];
+  }
+
   function sortedRows() {
     return state.rows.slice().sort((a, b) => {
-      const av = a[state.sortKey], bv = b[state.sortKey];
+      const av = cellValue(a, state.sortKey), bv = cellValue(b, state.sortKey);
       if (av == null || av === '') return 1;
       if (bv == null || bv === '') return -1;
       if (typeof av === 'number' && typeof bv === 'number') return (av - bv) * (state.sortDir === 'asc' ? 1 : -1);
@@ -317,8 +350,10 @@
     }).join('') + '</tr>';
     tbody.innerHTML = sortedRows().map((r) => '<tr>' + cols.map((c) => {
       const isNum = ['money', 'int', 'pct', 'num'].includes(c.type);
-      const cls = `col-${c.type}${isNum ? ' col-num' : ''}${c.type === 'pct' ? ' ' + rateClass(c.key, r[c.key]) : ''}`;
-      return `<td class="${cls}">${fmtCell(c, r[c.key])}</td>`;
+      const v = cellValue(r, c.key);
+      const display = (v != null && typeof v === 'object') ? JSON.stringify(v) : v;
+      const cls = `col-${c.type}${isNum ? ' col-num' : ''}${c.type === 'pct' ? ' ' + rateClass(c.key, v) : ''}`;
+      return `<td class="${cls}">${fmtCell(c, display)}</td>`;
     }).join('') + '</tr>').join('');
   }
 
@@ -337,7 +372,7 @@
     const { from, to } = currentRange();
     try {
       let url = state.mode === 'detail'
-        ? `/api/reports/daily?entity=${encodeURIComponent(entity)}&from=${from}&to=${to}&scope=shop`
+        ? `/api/reports/daily?entity=${encodeURIComponent(entity)}&from=${from}&to=${to}${state.rawView ? '' : '&scope=shop'}`
         : `/api/reports/aggregate?entity=${encodeURIComponent(entity)}&from=${from}&to=${to}&group=${state.mode}`;
       if (state.shopFilter.size) url += `&shops=${encodeURIComponent(Array.from(state.shopFilter).join(','))}`;
       if (state.q) url += `&q=${encodeURIComponent(state.q)}`;
@@ -550,6 +585,23 @@
 
   function exportCsv() {
     if (!state.entity) { showToast('暂无主体可导出', 'error'); return; }
+    // 全部字段模式:行数据只在客户端(raw 列由前端发现),就地生成 CSV
+    if (state.rawView) {
+      const cols = tableColumns();
+      const cell = (v) => {
+        const s = (v == null || typeof v !== 'object') ? String(v ?? '') : JSON.stringify(v);
+        return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
+      };
+      const lines = [cols.map((c) => cell(c.label)).join(',')];
+      for (const r of sortedRows()) lines.push(cols.map((c) => cell(cellValue(r, c.key))).join(','));
+      const blob = new Blob(['\ufeff' + lines.join('\n')], { type: 'text/csv;charset=utf-8' });
+      const a = document.createElement('a');
+      a.href = URL.createObjectURL(blob);
+      a.download = `湖明细-全部字段-${state.entity}.csv`;
+      a.click();
+      URL.revokeObjectURL(a.href);
+      return;
+    }
     const { from, to } = currentRange();
     let url = `/api/reports/export?entity=${encodeURIComponent(state.entity)}&from=${from}&to=${to}`;
     if (state.mode !== 'detail') url += `&group=${state.mode}`;
@@ -906,7 +958,8 @@
     state.shopFilter = new Set((a.shopIds || []).map(String));
     renderShopOptions();
     loadReport();
-    const nav = document.querySelector('.nav-item[data-view="reports"]');
+    // 明细浏览已挪到数据中台页,应用后跳到中台
+    const nav = document.querySelector('.nav-item[data-view="lake"]');
     if (nav) nav.click();
   }
 
@@ -1023,6 +1076,24 @@
       loadReport();
     });
 
+    // 字段模式:报表列(语义列)/ 全部字段(明细 + 全部原始字段;聚合视图无 raw,自动回明细)
+    $('reportFieldModes').addEventListener('click', (event) => {
+      const btn = event.target.closest('button[data-fields]');
+      if (!btn) return;
+      state.rawView = btn.dataset.fields === 'all';
+      $('reportFieldModes').querySelectorAll('button').forEach((b) => b.classList.toggle('active', b === btn));
+      if (state.rawView && state.mode !== 'detail') {
+        state.mode = 'detail';
+        state.sortKey = 'date';
+        state.sortDir = 'desc';
+        $('reportViewModes').querySelectorAll('button').forEach((b) => b.classList.toggle('active', b.dataset.mode === 'detail'));
+      }
+      $('reportViewModes').querySelectorAll('button').forEach((b) => { b.disabled = state.rawView; });
+      $('btnColumnCfg').hidden = state.rawView; // 列设置只作用于报表列模式
+      state.sortKey = state.rawView ? 'date' : state.sortKey;
+      renderTable();
+    });
+
     let searchTimer = null;
     $('reportSearch').addEventListener('input', () => {
       clearTimeout(searchTimer);
@@ -1102,6 +1173,45 @@
     });
   }
 
+  // ---------- 同步历史(数据采集页) ----------
+
+  function renderSyncHistory() {
+    const tbody = $('syncHistoryRows');
+    if (!tbody) return;
+    if (!state.entity) {
+      tbody.innerHTML = '<tr><td colspan="7" class="empty-cell">暂无主体</td></tr>';
+      return;
+    }
+    jsonFetch(`/api/reports/syncs?entity=${encodeURIComponent(state.entity)}`)
+      .then((data) => {
+        const runs = (data.runs || []).slice(0, 8);
+        if (!runs.length) {
+          tbody.innerHTML = '<tr><td colspan="7" class="empty-cell">暂无同步记录</td></tr>';
+          return;
+        }
+        const SCOPE_LABEL = { all: '全主体', chain: '按连锁', shop: '按门店' };
+        tbody.innerHTML = runs.map((r) => {
+          const warn = (r.steps || []).filter((s) => !s.ok).length;
+          const note = r.cancelled ? '已取消' : (r.failed ? `${r.failed} 失败` : (warn ? `${warn} 步骤告警(含全零)` : '成功'));
+          const cls = r.cancelled || r.failed ? 'badge-fail' : (warn ? 'badge-warn' : 'badge-ok');
+          const t = r.at ? new Date(r.at) : null;
+          const ts = t ? `${t.getMonth() + 1}/${t.getDate()} ${String(t.getHours()).padStart(2, '0')}:${String(t.getMinutes()).padStart(2, '0')}` : '–';
+          return `<tr>
+            <td>${ts}</td>
+            <td>${(r.from || '–').replace(/^(\d{4})(\d{2})(\d{2})$/, '$1-$2-$3')} ~ ${(r.to || '–').replace(/^(\d{4})(\d{2})(\d{2})$/, '$1-$2-$3')}</td>
+            <td>${SCOPE_LABEL[r.scope] || r.scope || '–'}</td>
+            <td class="num">${r.added ?? 0}</td>
+            <td class="num">${r.updated ?? 0}</td>
+            <td class="num">${r.failed ?? 0}</td>
+            <td><span class="badge ${cls}">${note}</span></td>
+          </tr>`;
+        }).join('');
+      })
+      .catch(() => {
+        tbody.innerHTML = '<tr><td colspan="7" class="empty-cell">同步历史加载失败</td></tr>';
+      });
+  }
+
   // ---------- 初始化 ----------
 
   // 数据采集页工具栏:推送当前主体到云端(本地湖 → 钉钉 AI 表格)
@@ -1153,6 +1263,7 @@
     }
     await refreshConn();
     refreshPushBadge();
+    renderSyncHistory();
     await loadReport();
   };
 

+ 7 - 0
server/routes/reports.js

@@ -50,6 +50,13 @@ function register(app, ctx) {
     sendJson(res, 200, { entities: naposService.store.listEntities() });
   });
 
+  // 同步历史(数据采集页「同步历史」面板)
+  app.get('/api/reports/syncs', async ({ res, query }) => {
+    const entity = query.entity || (naposService.store.listEntities()[0] || {}).key;
+    if (!entity) { sendJson(res, 200, { runs: [] }); return; }
+    sendJson(res, 200, { entity, runs: naposService.store.loadSyncLogs(entity) });
+  });
+
   // 数据中台:本地湖全景(主体覆盖、行数分布、最近同步/推送),供中台页渲染
   app.get('/api/reports/lake', async ({ res }) => {
     const store = naposService.store;