// sysutil/scan-ledger-template.ts
// scope: stream-safe Ledger template and size scan report

import path from "node:path";
import readline from "node:readline";
import { createReadStream } from "node:fs";
import { promises as fs } from "node:fs";
import { DataPaths } from "../pathList";

type LedgerRow = {
  timestamp?: number;
  recordID?: string;
  projectSerial?: string;
  source?: string;
  logType?: string;
  status?: string;
  filePath?: string;
  payload?: any;
  [key: string]: any;
};

type FileScan = {
  file: string;
  bytes: number;
  rows: number;
  invalidRows: number;
  byProjectSerial: Record<string, number>;
  byLogType: Record<string, number>;
  bySource: Record<string, number>;
  byStatus: Record<string, number>;
};

type LargeRow = {
  file: string;
  line: number;
  bytes: number;
  timestamp: number | null;
  recordID: string;
  projectSerial: string;
  source: string;
  logType: string;
  status: string;
  filePath: string;
  payloadKeys: string[];
};

const LEDGER_TEMPLATE = [
  { field: "timestamp", required: true, meaning: "numeric epoch milliseconds; global sort key" },
  { field: "recordID", required: true, meaning: "durable row/run reference" },
  { field: "projectSerial", required: true, meaning: "current stable project serial" },
  { field: "source", required: true, meaning: "entry source such as AUX, SYSTEM_CHAT_INPUT, AMP, RECORDER, AI" },
  { field: "logType", required: true, meaning: "MANIFEST, DIFF, AI_CALL, TAPE, SYSTEM, DIAGNOSTIC" },
  { field: "status", required: true, meaning: "recorded, done, error, failed, accepted, archived, etc." },
  { field: "filePath", required: false, meaning: "affected file path when applicable" },
  { field: "payload", required: true, meaning: "type-specific source content; may be compacted for large diffs" }
];

function ledgerCurrentFile(): string { return path.join(DataPaths.logsDir, "ledger-current.jsonl"); }
function ledgerArchiveDir(): string { return path.join(DataPaths.logsDir, "archive"); }
function normalizeKey(value: unknown, fallback: string): string {
  const clean = String(value ?? "").trim();
  return clean || fallback;
}
function addCount(target: Record<string, number>, key: string): void {
  target[key] = (target[key] || 0) + 1;
}
function rowInfo(file: string, line: number, bytes: number, row: LedgerRow): LargeRow {
  const payload = row.payload && typeof row.payload === "object" ? row.payload : {};
  return {
    file,
    line,
    bytes,
    timestamp: Number.isFinite(Number(row.timestamp)) ? Number(row.timestamp) : null,
    recordID: normalizeKey(row.recordID, "missing"),
    projectSerial: normalizeKey(row.projectSerial, "missing"),
    source: normalizeKey(row.source, "missing"),
    logType: normalizeKey(row.logType, "missing"),
    status: normalizeKey(row.status, "missing"),
    filePath: normalizeKey(row.filePath, ""),
    payloadKeys: Object.keys(payload).sort()
  };
}
function keepLargestRows(rows: LargeRow[], next: LargeRow, limit: number): void {
  rows.push(next);
  rows.sort((a, b) => b.bytes - a.bytes);
  if (rows.length > limit) rows.length = limit;
}
async function ledgerFiles(includeArchives: boolean): Promise<string[]> {
  const files = [ledgerCurrentFile()];
  if (!includeArchives) return files;
  try {
    const archives = (await fs.readdir(ledgerArchiveDir(), { withFileTypes: true }))
      .filter(entry => entry.isFile() && entry.name.endsWith(".jsonl"))
      .map(entry => path.join(ledgerArchiveDir(), entry.name))
      .sort((a, b) => b.localeCompare(a));
    files.push(...archives);
  } catch {}
  return files;
}
async function scanFile(file: string, largest: LargeRow[], largestLimit: number): Promise<FileScan> {
  const scan: FileScan = {
    file,
    bytes: 0,
    rows: 0,
    invalidRows: 0,
    byProjectSerial: {},
    byLogType: {},
    bySource: {},
    byStatus: {}
  };

  try {
    const stat = await fs.stat(file);
    scan.bytes = stat.size;
  } catch {
    return scan;
  }

  const rl = readline.createInterface({ input: createReadStream(file, { encoding: "utf8" }), crlfDelay: Infinity });
  let lineNo = 0;
  for await (const line of rl) {
    lineNo++;
    if (!line.trim()) continue;
    scan.rows++;
    const rowBytes = Buffer.byteLength(line, "utf8");
    try {
      const row = JSON.parse(line) as LedgerRow;
      addCount(scan.byProjectSerial, normalizeKey(row.projectSerial, "missing"));
      addCount(scan.byLogType, normalizeKey(row.logType, "missing"));
      addCount(scan.bySource, normalizeKey(row.source, "missing"));
      addCount(scan.byStatus, normalizeKey(row.status, "missing"));
      keepLargestRows(largest, rowInfo(file, lineNo, rowBytes, row), largestLimit);
    } catch {
      scan.invalidRows++;
      keepLargestRows(largest, {
        file,
        line: lineNo,
        bytes: rowBytes,
        timestamp: null,
        recordID: "invalid-json",
        projectSerial: "invalid-json",
        source: "invalid-json",
        logType: "invalid-json",
        status: "invalid-json",
        filePath: "",
        payloadKeys: []
      }, largestLimit);
    }
  }

  return scan;
}
function mergeCounts(target: Record<string, number>, source: Record<string, number>): void {
  for (const [key, value] of Object.entries(source)) target[key] = (target[key] || 0) + value;
}
async function main(): Promise<void> {
  const includeArchives = process.argv.includes("--all") || process.argv.includes("--archives");
  const limitArg = process.argv.find(arg => arg.startsWith("--largest="));
  const largestLimit = Math.max(1, Math.min(100, Math.floor(Number(limitArg?.split("=")[1] || 20)) || 20));
  const files = await ledgerFiles(includeArchives);
  const largestRows: LargeRow[] = [];
  const scans: FileScan[] = [];
  const totals = {
    files: 0,
    bytes: 0,
    rows: 0,
    invalidRows: 0,
    byProjectSerial: {} as Record<string, number>,
    byLogType: {} as Record<string, number>,
    bySource: {} as Record<string, number>,
    byStatus: {} as Record<string, number>
  };

  for (const file of files) {
    const scan = await scanFile(file, largestRows, largestLimit);
    scans.push(scan);
    if (scan.bytes <= 0 && scan.rows <= 0) continue;
    totals.files++;
    totals.bytes += scan.bytes;
    totals.rows += scan.rows;
    totals.invalidRows += scan.invalidRows;
    mergeCounts(totals.byProjectSerial, scan.byProjectSerial);
    mergeCounts(totals.byLogType, scan.byLogType);
    mergeCounts(totals.bySource, scan.bySource);
    mergeCounts(totals.byStatus, scan.byStatus);
  }

  console.log(JSON.stringify({
    ok: true,
    scanned: includeArchives ? "current-and-archives" : "current",
    template: LEDGER_TEMPLATE,
    totals,
    files: scans,
    largestRows
  }, null, 2));
}

main().catch(err => {
  console.error(err?.stack || err?.message || err);
  process.exit(1);
});
