Переглянути джерело

feat: 支持平台导出全字段明细导入(124 列)+ 重复校验

- ops-import 增加 platform-export 解析器:按「日期/门店编号/门店名称」定位,
  其余 100+ 列原样存入 data JSON;日期兼容 2026-10-18 / 2026/1/9 / 20261018
- 新表 ops_platform_row(主键 date+shop_id,SQLite/PG 双驱动)
- 上传时按「日期+门店」做重复校验:相同数据跳过、内容变化记冲突不覆盖,
  force 才覆盖;返回 newRows / identicalRows / conflictRows / duplicateData
- 学校归属:显式选择 > 文件名匹配已登记学校(含别名)> 报错要求补选
- 重新解析、db-export/db-import 同步支持平台行
- 测试:新增 8 条(解析/归一化/未选学校/首传/重传去重/冲突/强制覆盖)→ 42/42
liangzhao 2 тижнів тому
батько
коміт
437dc96143

+ 5 - 1
README.md

@@ -47,7 +47,11 @@
 
 采集之外的另一条数据入口:直接上传商家后台导出的人工统计表,不依赖商家客户端。
 
-- **支持格式**:一行一店的横表——`学校名称 / 店铺名称 / ID / 上线状态` + 每天 4 列(订单量 / 营收 / 曝光量 / 进店转换率)。日期列写成 `9.9订单量` 这种形式即可,年份在上传时选择。
+- **支持格式**(自动识别,两种可混着传):
+  - **人工统计横表**:`学校名称 / 店铺名称 / ID / 上线状态` + 每天 4 列(订单量 / 营收 / 曝光量 / 进店转换率),日期列写成 `9.9订单量`,年份在上传时选。
+  - **平台导出全字段明细**:一行一店一天,含 `日期 / 门店编号 / 门店名称` 与 100+ 列指标(营业额、有效订单、曝光、评分等),全部原样入库。
+- **重复校验**:平台明细按「日期 + 门店编号」判断——同一天同一店的相同数据自动跳过;内容有变化记为「冲突」不覆盖,勾选「覆盖已导入的文件」才覆盖。上传结果会显示「新增 N / 已存在 N / 冲突 N」。
+- **学校归属**:平台表里没有学校列时,优先用上传时选的学校,其次从文件名识别(如「浙江工商桐庐校区20261018…」匹配到已登记学校);都匹配不到会要求先选学校。
 - **原件存档**:每次上传的 xlsx 原样入库,可在「导入记录」里下载回;解析规则升级后可「重新解析」而不必重传。
 - **指标矩阵**:行=店铺(按学校分组)、列=日期,可按指标分区块查看;也支持卡片视图(区间合计 + 迷你趋势)。学校、日期、指标都可筛选,支持导出 CSV。
 - **脏数据容错**:`10..6%`、`12.3`(应为 12.3%)、`未开放`、空白等都会按规则清洗并带标记(表格里显示 ⚠,悬停看原始值),不会因为一格填错整批失败。

+ 93 - 4
ops-import.js

@@ -142,6 +142,78 @@ function detect(cells) {
   return metricHits >= 2 && hasRate && hasDateCol ? 'manual-ops' : null;
 }
 
+/** 平台导出(全字段明细):一行一店一天,含「日期 / 门店编号 / 门店名称」+ 100 多列指标 */
+function detectPlatformExport(cells) {
+  const header = ((cells && cells[0]) || []).map(normalizeHeader);
+  if (header.length < 50) return false;
+  const has = (name) => header.some((h) => h === name);
+  return has('日期') && has('门店编号') && has('门店名称');
+}
+
+/** 日期归一化:2026-10-18 / 2026/10/18 / 20261018 → 20261018 */
+function normalizeDate(value) {
+  const text = String(value == null ? '' : value).trim();
+  const m = /^(\d{4})[-/.](\d{1,2})[-/.](\d{1,2})/.exec(text);
+  if (m) return m[1] + pad2(Number(m[2])) + pad2(Number(m[3]));
+  const compact = text.replace(/\D/g, '');
+  return /^\d{8}$/.test(compact) ? compact : '';
+}
+
+function indexOfHeader(header, name) {
+  for (let i = 0; i < header.length; i++) if (normalizeHeader(header[i]) === name) return i;
+  return -1;
+}
+
+/**
+ * 平台导出 → 全字段行
+ * @param {Array<Array<any>>} cells
+ * @param {{ school?: string }} options
+ */
+function parsePlatformExport(cells, options = {}) {
+  const header = ((cells && cells[0]) || []).map((h) => String(h == null ? '' : h).trim());
+  const iDate = indexOfHeader(header, '日期');
+  const iShopId = indexOfHeader(header, '门店编号');
+  const iShopName = indexOfHeader(header, '门店名称');
+  if (iDate < 0 || iShopId < 0 || iShopName < 0) {
+    throw parseError('缺少「日期 / 门店编号 / 门店名称」列', 'OPS_FORMAT_UNSUPPORTED');
+  }
+  const school = String(options.school || '').trim();
+  const columns = header.map((h, i) => ({ name: h, col: i })).filter((c) => c.name);
+
+  const rows = [];
+  const shops = new Map();
+  const dates = new Set();
+  const warnings = [];
+
+  for (let r = 1; r < cells.length; r++) {
+    const line = cells[r] || [];
+    const date = normalizeDate(line[iDate]);
+    const shopId = toShopId(line[iShopId]);
+    const shopName = normalizeHeader(line[iShopName]);
+    if (!date || !shopId) {
+      const anyValue = line.some((v, i) => i !== iDate && v != null && v !== '');
+      if (anyValue) warnings.push({ row: r + 1, column: date ? '门店编号' : '日期', raw: date ? shopId : String(line[iDate] || ''), reason: 'missing-key' });
+      continue;
+    }
+    if (!shopName) {
+      warnings.push({ row: r + 1, column: '门店名称', raw: '', reason: 'missing-shop-name' });
+      continue;
+    }
+
+    const data = {};
+    for (const c of columns) {
+      const value = line[c.col];
+      data[c.name] = value === undefined ? null : value;
+    }
+
+    rows.push({ date, shopId, shopName, school, data });
+    dates.add(date);
+    if (!shops.has(shopId)) shops.set(shopId, { key: shopId, shopId, shopName, school, status: '', kind: 'platform' });
+  }
+
+  return { rows, shops: [...shops.values()], warnings, dates: [...dates].sort(), kind: 'platform-export' };
+}
+
 /**
  * 日期列 → [{ col, date, metricKey }]
  * 年份由调用方给定;列按出现顺序递增,遇到「月份比前一列小 6 以上」视为跨年。
@@ -304,13 +376,26 @@ const PARSERS = [
   { kind: 'manual-ops', detect, parse: parseManualOps },
 ];
 
+function detectKind(cells) {
+  if (detect(cells) === 'manual-ops') return 'manual-ops';
+  if (detectPlatformExport(cells)) return 'platform-export';
+  return null;
+}
+
 function detectBuffer(buffer) {
   const { sheets } = readXlsx(buffer);
   const cells = sheets.length ? sheets[0].rows : [];
-  for (const parser of PARSERS) {
-    if (parser.detect(cells)) return parser.kind;
-  }
-  return null;
+  return detectKind(cells);
+}
+
+/** 一步到位(按格式分流):Buffer → { kind, rows, shops, warnings, dates } */
+function parseAny(buffer, options = {}) {
+  const { sheets } = readXlsx(buffer);
+  const cells = sheets.length ? sheets[0].rows : [];
+  const kind = detectKind(cells);
+  if (kind === 'manual-ops') return { kind, ...parseManualOps(cells, options) };
+  if (kind === 'platform-export') return parsePlatformExport(cells, options);
+  throw parseError('暂不支持该表格格式', 'OPS_FORMAT_UNSUPPORTED');
 }
 
 module.exports = {
@@ -318,6 +403,10 @@ module.exports = {
   METRIC_KEYS,
   PARSERS,
   detect,
+  detectPlatformExport,
+  detectKind,
+  parseAny,
+  parsePlatformExport,
   detectBuffer,
   parseManualOps,
   parseWorkbook,

+ 1 - 0
ops-store.js

@@ -122,6 +122,7 @@ function createOpsStore(config) {
     getUpload: call('getUpload'),
     getContent: call('getContent'),
     queryRows: call('queryRows'),
+    queryPlatformRows: call('queryPlatformRows'),
     summary: call('summary'),
     get failure() {
       return lastFailure;

+ 8 - 1
public/ops.js

@@ -561,7 +561,14 @@
         } else {
           const r = body.report || {};
           item.status = 'done';
-          item.message = r.rows + ' 行 · ' + r.shops + ' 家';
+          if (r.kind === 'platform-export') {
+            const parts = ['新增 ' + r.newRows + ' 行'];
+            if (r.identicalRows) parts.push('已存在 ' + r.identicalRows + ' 行');
+            if (r.conflictRows) parts.push('冲突 ' + r.conflictRows + ' 行');
+            item.message = parts.join(' · ');
+          } else {
+            item.message = r.rows + ' 行 · ' + r.shops + ' 家';
+          }
           okCount += 1;
         }
       } catch (e) {

+ 3 - 1
scripts/db-export.js

@@ -35,6 +35,7 @@ const { createOpsStore } = require('../ops-store.js');
     }
   }
   const rows = await store.queryRows({});
+  const platformRows = await store.queryPlatformRows({});
   const summary = await store.summary();
 
   const payload = {
@@ -44,6 +45,7 @@ const { createOpsStore } = require('../ops-store.js');
     withFiles,
     uploads,
     shops: summary.shops,
+    platformRows,
     rows: rows.map((r) => ({
       date: r.date,
       shopKey: r.shopKey,
@@ -62,6 +64,6 @@ const { createOpsStore } = require('../ops-store.js');
   };
 
   fs.writeFileSync(outPath, JSON.stringify(payload, null, 2));
-  console.log(`已导出 ${uploads.length} 次上传、${rows.length} 行 → ${outPath}${withFiles ? '(含原件)' : '(不含原件)'}`);
+  console.log(`已导出 ${uploads.length} 次上传、${rows.length} 行人工数据、${platformRows.length} 行平台明细 → ${outPath}${withFiles ? '(含原件)' : '(不含原件)'}`);
   await store.close();
 })();

+ 10 - 0
scripts/db-import.js

@@ -58,6 +58,16 @@ const { createOpsStore } = require('../ops-store.js');
           content: Buffer.from(upload.contentBase64, 'base64'),
         });
       }
+      if (Array.isArray(payload.platformRows) && payload.platformRows.length) {
+        await tx.upsertPlatformRows(payload.platformRows.map((row) => ({
+          date: row.date,
+          shopId: row.shopId,
+          uploadId: row.uploadId,
+          school: row.school,
+          shopName: row.shopName,
+          data: row.data || {},
+        })));
+      }
       await tx.upsertRows(payload.rows.map((row) => ({
         date: row.date,
         shopKey: row.shopKey,

BIN
scripts/fixtures/platform-export-sample.xlsx


BIN
scripts/fixtures/platform-export-variant.xlsx


+ 80 - 0
scripts/test-ops-import.js

@@ -14,6 +14,8 @@ const crypto = require('crypto');
 
 const ROOT = path.resolve(__dirname, '..');
 const FIXTURE = path.join(__dirname, 'fixtures', 'ops-manual-sample.xlsx');
+const PLATFORM_FIXTURE = path.join(__dirname, 'fixtures', 'platform-export-sample.xlsx');
+const PLATFORM_VARIANT = path.join(__dirname, 'fixtures', 'platform-export-variant.xlsx');
 const importer = require(path.join(ROOT, 'ops-import.js'));
 
 const results = [];
@@ -64,6 +66,8 @@ async function readJson(res) {
 
 async function main() {
   const fixture = fs.readFileSync(FIXTURE);
+  const platformFixture = fs.readFileSync(PLATFORM_FIXTURE);
+  const platformVariant = fs.readFileSync(PLATFORM_VARIANT);
   const fixtureSha = crypto.createHash('sha256').update(fixture).digest('hex');
 
   // ---------- 解析器单测 ----------
@@ -165,6 +169,33 @@ async function main() {
     expect(parsed.shops.every((s) => s.shopName && s.shopName.trim()), '存在空店铺名');
   });
 
+  // ---------- 平台导出(全字段明细) ----------
+
+  await check('platform: 识别为 platform-export', () => {
+    expect(importer.detectBuffer(platformFixture) === 'platform-export', '识别失败');
+  });
+
+  const platformParsed = importer.parseAny(platformFixture, { school: '测试校区' });
+  await check('platform: 10 行 / 10 店 / 124 字段', () => {
+    expect(platformParsed.kind === 'platform-export', platformParsed.kind);
+    expect(platformParsed.rows.length === 10, `rows ${platformParsed.rows.length}`);
+    expect(platformParsed.shops.length === 10, `shops ${platformParsed.shops.length}`);
+    expect(Object.keys(platformParsed.rows[0].data).length === 124, `fields ${Object.keys(platformParsed.rows[0].data).length}`);
+    expect(platformParsed.rows[0].date === '20261018', platformParsed.rows[0].date);
+    expect(platformParsed.rows[0].school === '测试校区', platformParsed.rows[0].school);
+  });
+
+  await check('platform: 日期归一化', () => {
+    const cells = [
+      ['日期', '门店编号', '门店名称', '有效订单'],
+      ['2026/1/9', '1', 'A店', 1],
+      ['2026.10.18', '2', 'B店', 2],
+    ];
+    const r = importer.parsePlatformExport(cells, {});
+    expect(r.rows[0].date === '20260109', r.rows[0].date);
+    expect(r.rows[1].date === '20261018', r.rows[1].date);
+  });
+
   // ---------- 端到端(临时 SQLite) ----------
 
   const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ops-test-'));
@@ -322,6 +353,55 @@ async function main() {
       expect(sum.body.schools.length === 1 && sum.body.schools[0].name === '测试校区', JSON.stringify(sum.body.schools));
     });
 
+    const uploadPlatform = (buf, force, school) => fetch(base + '/api/ops/upload', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        fileName: '平台导出.xlsx',
+        contentBase64: buf.toString('base64'),
+        year: 2026,
+        force: !!force,
+        ...(school === undefined ? {} : { school }),
+      }),
+    }).then(readJson);
+
+    await check('E2E 平台导出:未选学校 → 400 提示补选', async () => {
+      const r = await uploadPlatform(platformFixture);
+      expect(r.status === 400 && r.body.code === 'OPS_SCHOOL_REQUIRED', `status ${r.status} ${JSON.stringify(r.body)}`);
+    });
+
+    await check('E2E 平台导出:首次导入 10 行', async () => {
+      const r = await uploadPlatform(platformFixture, false, '测试校区');
+      expect(r.status === 200, `status ${r.status} ${JSON.stringify(r.body).slice(0, 200)}`);
+      expect(r.body.report.kind === 'platform-export', r.body.report.kind);
+      expect(r.body.report.newRows === 10, `newRows ${r.body.report.newRows}`);
+      expect(r.body.report.writtenRows === 10, `written ${r.body.report.writtenRows}`);
+    });
+
+    await check('E2E 平台导出:同文件强制重传 → 全部已存在', async () => {
+      const r = await uploadPlatform(platformFixture, true, '测试校区');
+      expect(r.status === 200, `status ${r.status}`);
+      expect(r.body.report.newRows === 0, `newRows ${r.body.report.newRows}`);
+      expect(r.body.report.identicalRows === 10, `identical ${r.body.report.identicalRows}`);
+      expect(r.body.duplicateData === true, '应标记重复');
+    });
+
+    await check('E2E 平台导出:内容变化 → 冲突 1 行', async () => {
+      const r = await uploadPlatform(platformVariant, false, '测试校区');
+      expect(r.status === 200, `status ${r.status}`);
+      expect(r.body.report.conflictRows === 1, `conflict ${r.body.report.conflictRows}`);
+      expect(r.body.report.identicalRows === 9, `identical ${r.body.report.identicalRows}`);
+      expect(r.body.report.writtenRows === 0, `written ${r.body.report.writtenRows}`);
+    });
+
+    await check('E2E 平台导出:强制覆盖冲突', async () => {
+      const r = await uploadPlatform(platformVariant, true, '测试校区');
+      expect(r.status === 200, `status ${r.status}`);
+      expect(r.body.report.writtenRows === 10, `written ${r.body.report.writtenRows}`);
+      const sum = await fetch(base + '/api/ops/summary').then(readJson);
+      expect(sum.body.platformRowCount === 10, `platformRows ${sum.body.platformRowCount}`);
+    });
+
     await check('E2E 导入记录只含元数据', async () => {
       const r = await fetch(base + '/api/ops/uploads').then(readJson);
       expect(r.body.uploads.length >= 1, '记录为空');

+ 134 - 11
server/routes/ops.js

@@ -7,6 +7,7 @@ const crypto = require('crypto');
 const { sendJson, readBody } = require('../util.js');
 const opsImport = require('../../ops-import.js');
 const { createOpsStore } = require('../../ops-store.js');
+const { loadSchools } = require('../../schools-store.js');
 
 const MAX_UPLOAD_BYTES = 10 * 1024 * 1024;
 const UPLOAD_ID_RE = /^\d{8}-\d{6}-[a-z0-9]{6}$/;
@@ -113,6 +114,22 @@ function csvCell(value) {
   return /[",\n\r]/.test(text) ? '"' + text.replace(/"/g, '""') + '"' : text;
 }
 
+/** 从文件名识别已登记学校(取最长匹配,如「浙江工商桐庐校区20261018…」→ 浙江工商大学桐庐校区) */
+function resolveSchoolFromFileName(fileName) {
+  let best = '';
+  let bestName = '';
+  for (const school of loadSchools()) {
+    for (const candidate of [school.name, ...(school.aliases || [])]) {
+      const name = String(candidate || '').trim();
+      if (name && fileName.includes(name) && name.length > bestName.length) {
+        best = school.name;
+        bestName = name;
+      }
+    }
+  }
+  return best;
+}
+
 function register(app, ctx) {
   const store = createOpsStore(ctx.config);
 
@@ -147,16 +164,95 @@ function register(app, ctx) {
     }
 
     const year = Number(body.year) || new Date().getFullYear();
-    const school = String(body.school || '').trim().slice(0, 60);
+    let school = String(body.school || '').trim().slice(0, 60);
     let parsed;
     try {
-      parsed = opsImport.parseWorkbook(content, { year, school });
+      parsed = opsImport.parseAny(content, { year, school });
     } catch (e) {
       return sendJson(res, 400, { error: e.message || '解析失败', code: e.code || 'OPS_PARSE_FAILED' });
     }
     if (!parsed.rows.length) {
       return sendJson(res, 400, { error: '未解析到有效数据行' });
     }
+
+    const uploadId = newUploadId();
+    const uploadedAt = new Date().toISOString();
+    const from = parsed.dates[0] || '';
+    const to = parsed.dates[parsed.dates.length - 1] || '';
+
+    // ---------- 平台导出(全字段明细) ----------
+    if (parsed.kind === 'platform-export') {
+      // 表里没有学校列:优先用上传时选定的学校,其次从文件名识别(如「浙江工商桐庐校区20261018…」)
+      if (!school) school = resolveSchoolFromFileName(fileName);
+      if (!school) {
+        return sendJson(res, 400, { error: '请选择学校(平台导出表里没有学校列)', code: 'OPS_SCHOOL_REQUIRED' });
+      }
+
+      const existingRows = await store.queryPlatformRows({ from, to });
+      const known = new Map(existingRows.map((r) => [`${r.date}|${r.shopId}`, r]));
+      let newRows = 0;
+      let identicalRows = 0;
+      let conflictRows = 0;
+      const incoming = [];
+      for (const row of parsed.rows) {
+        const key = `${row.date}|${row.shopId}`;
+        const old = known.get(key);
+        if (!old) { newRows += 1; incoming.push(row); continue; }
+        const sameData = JSON.stringify(old.data) === JSON.stringify(row.data);
+        if (sameData && old.school === school) { identicalRows += 1; continue; }
+        conflictRows += 1;
+        if (body.force) incoming.push(row);
+      }
+      const toWrite = body.force ? parsed.rows : incoming;
+      const report = {
+        kind: 'platform-export',
+        rows: parsed.rows.length,
+        shops: parsed.shops.length,
+        newRows,
+        identicalRows,
+        conflictRows,
+        writtenRows: toWrite.length,
+        warningCount: parsed.warnings.length,
+        dates: { from, to },
+        year,
+        schoolOverride: school,
+        parser: { kind: 'platform-export', version: PARSER_VERSION },
+        warnings: parsed.warnings.slice(0, 500),
+      };
+
+      try {
+        await store.withTransaction(async (tx) => {
+          if (body.force) await tx.deletePlatformRowsBySha(sha256);
+          await tx.insertUpload({
+            id: uploadId,
+            originalName: fileName,
+            sha256,
+            sizeBytes: content.length,
+            uploadedAt,
+            year,
+            parserKind: 'platform-export',
+            parserVersion: PARSER_VERSION,
+            report,
+            content,
+          });
+          if (toWrite.length) {
+            await tx.upsertPlatformRows(toWrite.map((row) => ({ ...row, school, uploadId })));
+          }
+        });
+      } catch (e) {
+        return sendJson(res, 500, { error: '写入失败', detail: e.message || String(e) });
+      }
+
+      return sendJson(res, 200, {
+        ok: true,
+        uploadId,
+        report,
+        warnings: parsed.warnings.slice(0, 200),
+        duplicateData: report.newRows === 0 && report.conflictRows === 0,
+      });
+    }
+
+    // ---------- 人工统计表 ----------
     // 表格里没有「学校名称」列、上传时也没指定 → 让用户补选,避免落成空白分组
     if (!parsed.shops.some((shop) => shop.school && shop.school.trim())) {
       return sendJson(res, 400, {
@@ -165,20 +261,16 @@ function register(app, ctx) {
       });
     }
 
-    const uploadId = newUploadId();
-    const uploadedAt = new Date().toISOString();
-    const from = parsed.dates[0] || '';
-    const to = parsed.dates[parsed.dates.length - 1] || '';
-
     // 覆盖计数:该区间内已存在的「日期 + 店铺」组合
     let overwritten = 0;
     try {
-      const existingRows = await store.queryRows({ from, to });
-      const known = new Set(existingRows.map((r) => `${r.date}|${r.shopKey}`));
-      for (const row of parsed.rows) if (known.has(`${row.date}|${row.shopKey}`)) overwritten += 1;
+      const existingDaily = await store.queryRows({ from, to });
+      const knownDaily = new Set(existingDaily.map((r) => `${r.date}|${r.shopKey}`));
+      for (const row of parsed.rows) if (knownDaily.has(`${row.date}|${row.shopKey}`)) overwritten += 1;
     } catch { /* 计数失败不影响导入 */ }
 
     const report = {
+      kind: 'manual-ops',
       rows: parsed.rows.length,
       shops: parsed.shops.length,
       overwritten,
@@ -311,13 +403,44 @@ function register(app, ctx) {
     const school = (upload && upload.report && upload.report.schoolOverride) || '';
     let parsed;
     try {
-      parsed = opsImport.parseWorkbook(record.content, { year, school });
+      parsed = opsImport.parseAny(record.content, { year, school });
     } catch (e) {
       return sendJson(res, 400, { error: e.message || '解析失败', code: e.code || 'OPS_PARSE_FAILED' });
     }
     const from = parsed.dates[0] || '';
     const to = parsed.dates[parsed.dates.length - 1] || '';
+
+    if (parsed.kind === 'platform-export') {
+      const report = {
+        kind: 'platform-export',
+        rows: parsed.rows.length,
+        shops: parsed.shops.length,
+        newRows: parsed.rows.length,
+        identicalRows: 0,
+        conflictRows: 0,
+        writtenRows: parsed.rows.length,
+        warningCount: parsed.warnings.length,
+        dates: { from, to },
+        year,
+        schoolOverride: school,
+        parser: { kind: 'platform-export', version: PARSER_VERSION },
+        warnings: parsed.warnings.slice(0, 500),
+        reparsedAt: new Date().toISOString(),
+      };
+      try {
+        await store.withTransaction(async (tx) => {
+          await tx.deletePlatformRowsByUpload(params.id);
+          await tx.upsertPlatformRows(parsed.rows.map((row) => ({ ...row, school, uploadId: params.id })));
+          await tx.updateReport(params.id, report);
+        });
+      } catch (e) {
+        return sendJson(res, 500, { error: '重新解析写入失败', detail: e.message || String(e) });
+      }
+      return sendJson(res, 200, { ok: true, uploadId: params.id, report });
+    }
+
     const report = {
+      kind: 'manual-ops',
       rows: parsed.rows.length,
       shops: parsed.shops.length,
       overwritten: 0,

+ 52 - 1
server/stores/ops-pg.js

@@ -57,6 +57,17 @@ const DDL = [
      PRIMARY KEY (date, shop_key)
    )`,
   `CREATE INDEX IF NOT EXISTS ops_row_school_date_idx ON ops_row (school, date)`,
+  `CREATE TABLE IF NOT EXISTS ops_platform_row (
+     date char(8) NOT NULL,
+     shop_id text NOT NULL,
+     upload_id text NOT NULL REFERENCES ops_upload(id) ON DELETE CASCADE,
+     school text NOT NULL DEFAULT '',
+     shop_name text NOT NULL DEFAULT '',
+     data jsonb NOT NULL DEFAULT '{}'::jsonb,
+     updated_at timestamptz NOT NULL,
+     PRIMARY KEY (date, shop_id)
+   )`,
+  `CREATE INDEX IF NOT EXISTS ops_platform_row_date_idx ON ops_platform_row (date)`,
 ];
 
 function rowToApi(row) {
@@ -194,6 +205,25 @@ function createPgStore({ connectionString, ssl = true, max = 5, schema = 'public
       async deleteRowsByUpload(uploadId) {
         await client.query('DELETE FROM ops_row WHERE upload_id = $1', [uploadId]);
       },
+      async upsertPlatformRows(rows) {
+        const now = new Date().toISOString();
+        for (const r of rows) {
+          await client.query(
+            `INSERT INTO ops_platform_row (date, shop_id, upload_id, school, shop_name, data, updated_at)
+             VALUES ($1,$2,$3,$4,$5,$6,$7)
+             ON CONFLICT (date, shop_id) DO UPDATE SET
+               upload_id = EXCLUDED.upload_id, school = EXCLUDED.school,
+               shop_name = EXCLUDED.shop_name, data = EXCLUDED.data, updated_at = EXCLUDED.updated_at`,
+            [r.date, r.shopId, r.uploadId, r.school || '', r.shopName || '', JSON.stringify(r.data || {}), now],
+          );
+        }
+      },
+      async deletePlatformRowsByUpload(uploadId) {
+        await client.query('DELETE FROM ops_platform_row WHERE upload_id = $1', [uploadId]);
+      },
+      async deletePlatformRowsBySha(sha256) {
+        await client.query('DELETE FROM ops_platform_row WHERE upload_id IN (SELECT id FROM ops_upload WHERE sha256 = $1)', [sha256]);
+      },
       async deleteRowsBySha(sha256) {
         await client.query('DELETE FROM ops_row WHERE upload_id IN (SELECT id FROM ops_upload WHERE sha256 = $1)', [sha256]);
       },
@@ -261,11 +291,30 @@ function createPgStore({ connectionString, ssl = true, max = 5, schema = 'public
     return r.rows.map(rowToApi);
   }
 
+  async function queryPlatformRows({ from, to } = {}) {
+    const where = [];
+    const params = [];
+    if (from) { params.push(String(from)); where.push(`date >= ${params.length}`); }
+    if (to) { params.push(String(to)); where.push(`date <= ${params.length}`); }
+    const sql = `SELECT * FROM ops_platform_row ${where.length ? 'WHERE ' + where.join(' AND ') : ''} ORDER BY date ASC, shop_id ASC`;
+    const r = await getPool().query(sql, params);
+    return r.rows.map((row) => ({
+      date: String(row.date).trim(),
+      shopId: row.shop_id,
+      uploadId: row.upload_id,
+      school: row.school,
+      shopName: row.shop_name,
+      data: row.data || {},
+      updatedAt: row.updated_at instanceof Date ? row.updated_at.toISOString() : row.updated_at,
+    }));
+  }
+
   async function summary() {
     const poolRef = getPool();
-    const [uploadCount, rowCount, range, schools, shops, last] = await Promise.all([
+    const [uploadCount, rowCount, platformRowCount, range, schools, shops, last] = await Promise.all([
       poolRef.query('SELECT COUNT(*)::int AS c FROM ops_upload'),
       poolRef.query('SELECT COUNT(*)::int AS c FROM ops_row'),
+      poolRef.query('SELECT COUNT(*)::int AS c FROM ops_platform_row'),
       poolRef.query('SELECT MIN(date) AS f, MAX(date) AS t FROM ops_row'),
       poolRef.query('SELECT school, COUNT(*)::int AS c FROM ops_shop GROUP BY school ORDER BY school ASC'),
       poolRef.query('SELECT key, shop_id, shop_name, school, status FROM ops_shop ORDER BY school ASC, shop_name ASC'),
@@ -276,6 +325,7 @@ function createPgStore({ connectionString, ssl = true, max = 5, schema = 'public
       shops: shops.rows.map((r) => ({ key: r.key, shopId: r.shop_id, shopName: r.shop_name, school: r.school, status: r.status })),
       uploadCount: uploadCount.rows[0].c,
       rowCount: rowCount.rows[0].c,
+      platformRowCount: platformRowCount.rows[0].c,
       coverage: range.rows[0] && range.rows[0].f ? { from: String(range.rows[0].f).replace(/-/g, ''), to: String(range.rows[0].t).replace(/-/g, '') } : null,
       lastUpload: last.rows.length ? {
         id: last.rows[0].id,
@@ -301,6 +351,7 @@ function createPgStore({ connectionString, ssl = true, max = 5, schema = 'public
     health,
     withTransaction,
     findUploadBySha,
+    queryPlatformRows,
     listUploads,
     getUpload,
     getContent,

+ 56 - 1
server/stores/ops-sqlite.js

@@ -58,6 +58,17 @@ const DDL = [
      PRIMARY KEY (date, shop_key)
    )`,
   `CREATE INDEX IF NOT EXISTS ops_row_school_date_idx ON ops_row (school, date)`,
+  `CREATE TABLE IF NOT EXISTS ops_platform_row (
+     date TEXT NOT NULL,
+     shop_id TEXT NOT NULL,
+     upload_id TEXT NOT NULL REFERENCES ops_upload(id) ON DELETE CASCADE,
+     school TEXT NOT NULL DEFAULT '',
+     shop_name TEXT NOT NULL DEFAULT '',
+     data TEXT NOT NULL DEFAULT '{}',
+     updated_at TEXT NOT NULL,
+     PRIMARY KEY (date, shop_id)
+   )`,
+  `CREATE INDEX IF NOT EXISTS ops_platform_row_date_idx ON ops_platform_row (date)`,
 ];
 
 function rowToApi(row) {
@@ -207,6 +218,47 @@ function createSqliteStore({ file }) {
     open().prepare('DELETE FROM ops_row WHERE upload_id = ?').run(uploadId);
   }
 
+  function upsertPlatformRows(rows) {
+    const stmt = open().prepare(`INSERT INTO ops_platform_row
+      (date, shop_id, upload_id, school, shop_name, data, updated_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(date, shop_id) DO UPDATE SET
+        upload_id = excluded.upload_id,
+        school = excluded.school,
+        shop_name = excluded.shop_name,
+        data = excluded.data,
+        updated_at = excluded.updated_at`);
+    const now = new Date().toISOString();
+    for (const r of rows) {
+      stmt.run(r.date, r.shopId, r.uploadId, r.school || '', r.shopName || '', JSON.stringify(r.data || {}), now);
+    }
+  }
+
+  function deletePlatformRowsByUpload(uploadId) {
+    open().prepare('DELETE FROM ops_platform_row WHERE upload_id = ?').run(uploadId);
+  }
+
+  function deletePlatformRowsBySha(sha256) {
+    open().prepare('DELETE FROM ops_platform_row WHERE upload_id IN (SELECT id FROM ops_upload WHERE sha256 = ?)').run(sha256);
+  }
+
+  async function queryPlatformRows({ from, to } = {}) {
+    const where = [];
+    const params = [];
+    if (from) { where.push('date >= ?'); params.push(String(from)); }
+    if (to) { where.push('date <= ?'); params.push(String(to)); }
+    const sql = `SELECT * FROM ops_platform_row ${where.length ? 'WHERE ' + where.join(' AND ') : ''} ORDER BY date ASC, shop_id ASC`;
+    return open().prepare(sql).all(...params).map((row) => ({
+      date: row.date,
+      shopId: row.shop_id,
+      uploadId: row.upload_id,
+      school: row.school,
+      shopName: row.shop_name,
+      data: safeJson(row.data, {}),
+      updatedAt: row.updated_at,
+    }));
+  }
+
   function deleteRowsBySha(sha256) {
     open().prepare('DELETE FROM ops_row WHERE upload_id IN (SELECT id FROM ops_upload WHERE sha256 = ?)').run(sha256);
   }
@@ -223,7 +275,7 @@ function createSqliteStore({ file }) {
     const handle = open();
     handle.exec('BEGIN IMMEDIATE');
     try {
-      const result = await fn({ insertUpload, upsertRows, upsertShops, deleteRowsByUpload, deleteRowsBySha, pruneOrphanShops, updateReport });
+      const result = await fn({ insertUpload, upsertRows, upsertShops, upsertPlatformRows, deleteRowsByUpload, deleteRowsBySha, deletePlatformRowsByUpload, deletePlatformRowsBySha, pruneOrphanShops, updateReport });
       handle.exec('COMMIT');
       // 写事务结束后做一次 checkpoint:让 data/ops.db 自包含,备份时不必关心 -wal
       try { handle.exec('PRAGMA wal_checkpoint(TRUNCATE)'); } catch { /* 有读事务时跳过 */ }
@@ -297,6 +349,7 @@ function createSqliteStore({ file }) {
     const handle = open();
     const uploadCount = handle.prepare('SELECT COUNT(*) AS c FROM ops_upload').get().c;
     const rowCount = handle.prepare('SELECT COUNT(*) AS c FROM ops_row').get().c;
+    const platformRowCount = handle.prepare('SELECT COUNT(*) AS c FROM ops_platform_row').get().c;
     const range = handle.prepare('SELECT MIN(date) AS f, MAX(date) AS t FROM ops_row').get();
     const schools = handle.prepare('SELECT school, COUNT(*) AS c FROM ops_shop GROUP BY school ORDER BY school ASC')
       .all().map((r) => ({ name: r.school, shopCount: r.c }));
@@ -309,6 +362,7 @@ function createSqliteStore({ file }) {
       shops,
       uploadCount,
       rowCount,
+      platformRowCount,
       coverage: range && range.f ? { from: range.f, to: range.t } : null,
       lastUpload: last ? {
         id: last.id,
@@ -341,6 +395,7 @@ function createSqliteStore({ file }) {
     deleteRowsByUpload,
     updateReport,
     findUploadBySha,
+    queryPlatformRows,
     listUploads,
     getUpload,
     getContent,