DriftLedger/tests/importRealFiles.test.ts

import { describe, it, expect } from 'vitest';
import fs from 'fs';
import path from 'path';
import * as XLSX from 'xlsx';
import { importStatement } from '../src/domain/statements';
import { batchDedup } from '../src/domain/dedup';

// @ts-ignore
const GbkTextDecoder = (() => {
  const origDecoder = global.TextDecoder;
  const origEncoder = global.TextEncoder;
  global.TextDecoder = undefined as any;
  global.TextEncoder = undefined as any;
  const dec = require('text-encoding-gbk').TextDecoder;
  global.TextDecoder = origDecoder;
  global.TextEncoder = origEncoder;
  return dec;
})();

describe('Real Files Import Integrity and Health Test', () => {
  it('should parse both files and pass all field validation checks', () => {
    const exampleDir = path.join(__dirname, '../example');
    const files = fs.readdirSync(exampleDir);

    const xlsxFileName = files.find(f => f.endsWith('.xlsx') && !f.startsWith('~$'));
    const csvFileName = files.find(f => f.endsWith('.csv') && !f.startsWith('~$'));

    expect(xlsxFileName).toBeDefined();
    expect(csvFileName).toBeDefined();

    // 1. Parse WeChat Excel file
    const xlsxPath = path.join(exampleDir, xlsxFileName!);
    const workbook = XLSX.readFile(xlsxPath);
    const sheetName = workbook.SheetNames[0];
    const worksheet = workbook.Sheets[sheetName];
    const wechatCsv = XLSX.utils.sheet_to_csv(worksheet);
    const wechatEvents = importStatement(wechatCsv, 'wechat-csv-v1');

    // 2. Parse Alipay CSV file
    const csvPath = path.join(exampleDir, csvFileName!);
    const csvBuffer = fs.readFileSync(csvPath);
    const decoder = new GbkTextDecoder('gbk');
    const alipayCsv = decoder.decode(csvBuffer);
    const alipayEvents = importStatement(alipayCsv, 'alipay-csv-v1');

    const allEvents = [...wechatEvents, ...alipayEvents];
    console.log(`Total events to validate: ${allEvents.length}`);

    const seenIds = new Set<string>();

    for (let i = 0; i < allEvents.length; i++) {
      const e = allEvents[i];
      const errMsg = `Failed validation at index ${i} (Channel: ${e.channel}, ID: ${e.id})`;

      // A. ID Validation
      expect(e.id, `${errMsg}: ID should be a non-empty string`).toBeTruthy();
      expect(typeof e.id, `${errMsg}: ID should be a string`).toBe('string');
      expect(seenIds.has(e.id), `${errMsg}: ID ${e.id} is duplicated!`).toBe(false);
      seenIds.add(e.id);

      // B. Date and Time Format Validation
      expect(e.occurredAt, `${errMsg}: occurredAt should be a non-empty string`).toBeTruthy();
      // Date should match YYYY-MM-DD format prefix
      expect(e.occurredAt, `${errMsg}: occurredAt "${e.occurredAt}" does not start with YYYY-MM-DD`).toMatch(/^\d{4}-\d{2}-\d{2}/);
      // Parsing the date should be valid (not NaN)
      const parsedTime = Date.parse(e.occurredAt);
      expect(Number.isNaN(parsedTime), `${errMsg}: occurredAt "${e.occurredAt}" is not a valid date string`).toBe(false);

      // C. Amount Format Validation
      expect(e.amount, `${errMsg}: amount should be a non-empty string`).toBeTruthy();
      // Amount must be a valid signed decimal string
      expect(e.amount, `${errMsg}: amount "${e.amount}" is not a valid signed decimal string`).toMatch(/^-?\d+(\.\d+)?$/);
      const parsedAmount = parseFloat(e.amount);
      expect(Number.isNaN(parsedAmount), `${errMsg}: amount "${e.amount}" cannot be parsed into a number`).toBe(false);

      // D. Currency Code Validation
      expect(e.currency, `${errMsg}: currency should be a non-empty string`).toBeTruthy();
      expect(e.currency, `${errMsg}: currency "${e.currency}" is invalid`).toMatch(/^[A-Z]{3}$/);

      // E. Direction Validation
      expect(e.direction, `${errMsg}: direction "${e.direction}" is invalid`).toMatch(/^(income|expense|transfer|refund|fee)$/);

      // F. Channel Code Validation
      expect(e.channel, `${errMsg}: channel "${e.channel}" is invalid`).toMatch(/^(Alipay|WeChat|Bank)$/);

      // G. Column Shift / Misalignment Check
      // Ensure data has not shifted (e.g. headers appeared as values)
      const headerKeywords = ['金额', '交易时间', '交易对方', '收/支', '商品', '备注', '交易单号', '商户单号'];
      headerKeywords.forEach(kw => {
        expect(e.amount.includes(kw), `${errMsg}: amount "${e.amount}" contains header keyword "${kw}"`).toBe(false);
        expect(e.occurredAt.includes(kw), `${errMsg}: occurredAt "${e.occurredAt}" contains header keyword "${kw}"`).toBe(false);
      });
    }

    console.log('✓ All 468 events passed structural integrity checks!');

    // 3. Run Deduplication
    const { accepted, duplicates } = batchDedup(allEvents);
    console.log(`✓ Deduplication executed: ${accepted.length} accepted, ${duplicates.length} duplicates.`);

    // 4. Verify specific test cases (Caitang refund and Metro rides)
    const caitangEvents = accepted.filter(e => e.counterparty.includes('彩棠'));
    expect(caitangEvents.some(e => (e.direction === 'refund' || e.direction === 'income') && parseFloat(e.amount) === 238)).toBe(true);
    expect(caitangEvents.some(e => e.direction === 'expense' && parseFloat(e.amount) === -238)).toBe(true);

    const metroEvents = accepted.filter(e => e.counterparty.includes('长沙地铁') && e.occurredAt.startsWith('2026-04-25'));
    expect(metroEvents.length).toBe(2);

    console.log('✓ Refund and Metro rides specific test cases passed!');
  });
});