// sysutil/scrape.ts
// Utility: create, list, and restore scrapes from the command line.
// A scrape is a flat text file of restorable source files.

import { promises as fs } from "node:fs";
import path from "node:path";
import { ROOT, SETTINGS_PATH } from "../pathList";

const BACKSTAGE_SERIAL = "00";

const RESTORABLE_EXTENSIONS = new Set([".ts", ".js", ".css", ".html", ".txt", ".md"]);

const GLOBAL_EXCLUDED_DIRS = new Set([
  "node_modules", ".git", ".temp", "tmp", "logs", "scrapes", "scrape", 
  "backups", "assets", "user", "users", "secure", "localsecure", "dist", "build"
]);

const EXCLUDED_FILES = new Set(["package-lock.json", "settings.json", "user.json"]);
const EXCLUDED_NAME_CONTAINS = ["scrape", "restore"];

// ── Types ─────────────────────────────────────────────────────────────────────

export type ScrapeResult = {
  ok: boolean;
  scrapeFile: string;
  fileCount: number;
  byteCount: number;
  serial: string;
  projectName: string;
};

export type ScrapeListEntry = {
  name: string;
  path: string;
  serial: string;
  createdApprox: string;
};

export type ScrapeRestoreResult = {
  ok: boolean;
  scrapeFile: string;
  restoredCount: number;
  unchangedCount: number;
  files: Array<{ filePath: string; action: "restored" | "created" | "unchanged" }>;
};

// ── Utilities ─────────────────────────────────────────────────────────────────

function normalizeSlashes(v: string): string { return String(v || "").replace(/\\/g, "/"); }
function pad(n: number): string { return String(n).padStart(2, "0"); }

function tsCompact(): string {
  const d = new Date();
  return `${d.getFullYear()}-${pad(d.getMonth()+1)}-${pad(d.getDate())}_${pad(d.getHours())}-${pad(d.getMinutes())}-${pad(d.getSeconds())}`;
}

async function exists(fp: string): Promise<boolean> {
  try { await fs.access(fp); return true; } catch { return false; }
}

async function readJsonSafe<T>(fp: string, fallback: T): Promise<T> {
  try { return JSON.parse(await fs.readFile(fp, "utf8")) as T; } catch { return fallback; }
}

function relativeFromRoot(absPath: string): string {
  return normalizeSlashes(path.relative(ROOT, absPath));
}

function isRestorable(name: string): boolean {
  const clean = String(name || "").trim().toLowerCase();
  if (!clean) return false;
  if (EXCLUDED_FILES.has(clean)) return false;
  if (clean.endsWith(".log")) return false;
  if (EXCLUDED_NAME_CONTAINS.some(p => clean.includes(p))) return false;
  return RESTORABLE_EXTENSIONS.has(path.extname(clean));
}

function shouldSkipDir(name: string): boolean {
  return GLOBAL_EXCLUDED_DIRS.has(String(name || "").trim().toLowerCase());
}

function log(msg: string): void  { console.log(`[scrape] ${msg}`); }
function warn(msg: string): void { console.warn(`[scrape:warn] ${msg}`); }

// ── Settings helpers ──────────────────────────────────────────────────────────

type ProjectRecord = { name: string; location?: string; path?: string; serialNo?: string };

async function readSettings(): Promise<{ projects: ProjectRecord[]; activeProject: string }> {
  const s = await readJsonSafe<any>(SETTINGS_PATH, {});
  return {
    projects:       Array.isArray(s?.projects) ? s.projects : [],
    activeProject: String(s?.activeProject || "").trim()
  };
}

async function resolveProjectForScrape(opts: { project?: string; pathArg?: string; system?: boolean }): Promise<{ name: string; serial: string; root: string }> {
  if (opts.system) {
    return { name: "system", serial: BACKSTAGE_SERIAL, root: ROOT };
  }

  const { projects, activeProject } = await readSettings();

  if (opts.pathArg) {
    const abs = path.isAbsolute(opts.pathArg) ? path.resolve(opts.pathArg) : path.resolve(ROOT, opts.pathArg);
    const match = projects.find(p => {
      const loc = String(p.location || p.path || "").trim();
      if (!loc) return false;
      const root = path.isAbsolute(loc) ? path.resolve(loc) : path.resolve(ROOT, loc);
      return root === abs;
    });
    return {
      name:   match?.name || path.basename(abs),
      serial: String(match?.serialNo || "").trim() || BACKSTAGE_SERIAL,
      root:   abs
    };
  }

  const needle = opts.project || activeProject;
  if (!needle) throw new Error("No project selected. Select a project first, or use --project <name>, --path <dir>, or --system.");

  const match = projects.find(p =>
    p.name === needle ||
    p.name.toLowerCase() === needle.toLowerCase() ||
    String(p.serialNo || "").trim() === needle
  );

  if (!match) throw new Error(`Project not found: "${needle}". Use --list to see available projects.`);

  const loc = String(match.location || match.path || "").trim();
  if (!loc)  throw new Error(`Project "${match.name}" has no location.`);

  return {
    name:   match.name,
    serial: String(match.serialNo || "").trim() || BACKSTAGE_SERIAL,
    root:   path.isAbsolute(loc) ? path.resolve(loc) : path.resolve(ROOT, loc)
  };
}

// ── Scrape directory for scope ────────────────────────────────────────────────

function getScrapeDir(serial: string): string {
  const scope = serial === BACKSTAGE_SERIAL ? "system" : serial;
  return path.join(ROOT, "logs", scope, "scrapes");
}

// ── Walk ──────────────────────────────────────────────────────────────────────

async function walkRestorable(absRoot: string, visited = new Set<string>()): Promise<Array<{ abs: string; rel: string }>> {
  const abs = path.resolve(absRoot);
  if (visited.has(abs)) return [];
  visited.add(abs);

  let stat;
  try { stat = await fs.stat(abs); } catch { return []; }

  if (stat.isFile()) {
    if (!isRestorable(path.basename(abs))) return [];
    return [{ abs, rel: relativeFromRoot(abs) }];
  }

  if (!stat.isDirectory()) return [];
  if (shouldSkipDir(path.basename(abs))) return [];

  let entries: string[];
  try { entries = await fs.readdir(abs); } catch { return []; }

  const results: Array<{ abs: string; rel: string }> = [];
  for (const entry of entries.sort()) {
    const childAbs = path.join(abs, entry);
    results.push(...await walkRestorable(childAbs, visited));
  }
  return results;
}

// ── Create scrape ─────────────────────────────────────────────────────────────

export async function createScrape(opts: {
  project?: string;
  pathArg?: string;
  system?: boolean;
  label?: string;
} = {}): Promise<ScrapeResult> {
  const scope = await resolveProjectForScrape(opts);

  if (!(await exists(scope.root))) {
    throw new Error(`Root path does not exist: ${scope.root}`);
  }

  const files = await walkRestorable(scope.root);
  if (!files.length) throw new Error(`No restorable files found under ${scope.root}`);

  const parts: string[] = [];
  let byteCount = 0;

  for (const { abs, rel } of files) {
    try {
      const content = await fs.readFile(abs, "utf8");
      const section = `/${rel}\n${content}\n`;
      parts.push(section);
      byteCount += Buffer.byteLength(section, "utf8");
    } catch {
      warn(`Could not read ${rel}, skipping`);
    }
  }

  // Determine naming: SYSTEM or directory under /projects/
  let namingPart = scope.name;
  if (opts.system) {
    namingPart = "SYSTEM";
  } else {
    const parts = normalizeSlashes(scope.root).split('/');
    const idx = parts.lastIndexOf('projects');
    if (idx !== -1 && idx < parts.length - 1) {
      namingPart = parts[idx + 1];
    }
  }

  const label      = opts.label ? `_${opts.label}` : "";
  const scrapeName = `scrape_${namingPart.replace(/[^a-z0-9_-]/gi, "_")}_${tsCompact()}${label}.txt`;
  const scrapeDir  = getScrapeDir(scope.serial);
  const scrapePath = path.join(scrapeDir, scrapeName);

  await fs.mkdir(scrapeDir, { recursive: true });
  await fs.writeFile(scrapePath, parts.join("\n"), "utf8");

  log(`Created: ${normalizeSlashes(path.relative(ROOT, scrapePath))}`);
  log(`  ${files.length} files, ${(byteCount / 1024).toFixed(1)} KB`);

  return {
    ok:            true,
    scrapeFile:  normalizeSlashes(path.relative(ROOT, scrapePath)),
    fileCount:   files.length,
    byteCount,
    serial:      scope.serial,
    projectName: scope.name
  };
}

// ── List scrapes ──────────────────────────────────────────────────────────────

export async function listScrapes(opts: {
  project?: string;
  pathArg?: string;
  system?: boolean;
  all?: boolean;
} = {}): Promise<ScrapeListEntry[]> {
  const entries: ScrapeListEntry[] = [];

  if (opts.all) {
    try {
      const serialDirs = await fs.readdir(path.join(ROOT, "logs"), { withFileTypes: true });
      for (const d of serialDirs) {
        if (!d.isDirectory()) continue;
        entries.push(...await listScrapesForSerial(d.name));
      }
    } catch {}
  } else {
    const scope = await resolveProjectForScrape(opts).catch(() => null);
    if (scope) entries.push(...await listScrapesForSerial(scope.serial));
  }

  return entries.sort((a, b) => b.name.localeCompare(a.name));
}

async function listScrapesForSerial(serial: string): Promise<ScrapeListEntry[]> {
  const scrapeDir = getScrapeDir(serial);
  try {
    const files = await fs.readdir(scrapeDir, { withFileTypes: true });
    return files
      .filter(f => f.isFile() && f.name.endsWith(".txt"))
      .map(f => ({
        name:        f.name,
        path:        normalizeSlashes(path.relative(ROOT, path.join(scrapeDir, f.name))),
        serial,
        createdApprox: f.name.replace(/^scrape_[^_]+_/, "").replace(/\.txt$/, "").replace(/_/g, " ").replace(/-/g, ":").replace(/(\d{4}:\d{2}:\d{2}) /, "$1 ")
      }));
  } catch {
    return [];
  }
}

// ── Restore from scrape ───────────────────────────────────────────────────────

function isScrapePathHeader(line: string): boolean {
  const c = String(line || "").trim();
  return Boolean(c) && c.startsWith("/") && RESTORABLE_EXTENSIONS.has(path.extname(c).toLowerCase());
}

async function parseScrape(scrapePath: string): Promise<Map<string, string>> {
  const lines = (await fs.readFile(scrapePath, "utf8")).split(/\r?\n/);
  const out   = new Map<string, string>();
  let i = 0;

  while (i < lines.length) {
    if (!isScrapePathHeader(lines[i])) { i++; continue; }
    const relPath = lines[i].trim().replace(/^\//, "");
    const body: string[] = [];
    i++;
    while (i < lines.length) {
      if (isScrapePathHeader(lines[i])) break;
      if (body.length >= 2 && !body[body.length-1] && !body[body.length-2] && !lines[i].trim()) { i++; break; }
      body.push(lines[i]); i++;
    }
    out.set(normalizeSlashes(relPath), body.join("\n").replace(/\n{2,}$/g, "\n").replace(/\n$/, ""));
  }
  return out;
}

export async function restoreFromScrapeFile(name: string, dryRun = false): Promise<ScrapeRestoreResult> {
  let scrapePath: string | null = null;

  if (name.includes("/") || name.includes("\\")) {
    const abs = path.isAbsolute(name) ? name : path.resolve(ROOT, name);
    if (await exists(abs)) scrapePath = abs;
  }

  if (!scrapePath) {
    try {
      const serialDirs = await fs.readdir(path.join(ROOT, "logs"), { withFileTypes: true });
      for (const d of serialDirs) {
        if (!d.isDirectory()) continue;
        const candidate = path.join(ROOT, "logs", d.name, "scrapes", name);
        if (await exists(candidate)) { scrapePath = candidate; break; }
      }
    } catch {}
  }


  if (!scrapePath) throw new Error(`Scrape file not found: "${name}"`);

  const stat   = await fs.stat(scrapePath);
  const parsed = await parseScrape(scrapePath);

  if (!parsed.size) throw new Error(`No restorable files found in scrape: ${name}`);

  const results: Array<{ filePath: string; action: "restored" | "created" | "unchanged" }> = [];

  for (const [relPath, content] of parsed.entries()) {
    const abs = path.resolve(ROOT, relPath);

    if (!abs.startsWith(path.resolve(ROOT) + path.sep) && abs !== path.resolve(ROOT)) {
      warn(`Skipping path outside ROOT: ${relPath}`);
      continue;
    }

    if (!(await exists(abs))) {
      if (!dryRun) {
        await fs.mkdir(path.dirname(abs), { recursive: true });
        await fs.writeFile(abs, content, "utf8");
      }
      results.push({ filePath: relPath, action: "created" });
      continue;
    }

    const current = await fs.readFile(abs, "utf8");
    if (current === content) { results.push({ filePath: relPath, action: "unchanged" }); continue; }

    let shouldWrite = true;
    try { const s = await fs.stat(abs); shouldWrite = s.mtimeMs >= stat.mtimeMs; } catch {}

    if (!shouldWrite) { results.push({ filePath: relPath, action: "unchanged" }); continue; }

    if (!dryRun) {
      await fs.mkdir(path.dirname(abs), { recursive: true });
      await fs.writeFile(abs, content, "utf8");
    }
    results.push({ filePath: relPath, action: "restored" });
  }

  const restoredCount  = results.filter(r => r.action !== "unchanged").length;
  const unchangedCount = results.filter(r => r.action === "unchanged").length;

  return {
    ok: true,
    scrapeFile:    normalizeSlashes(path.relative(ROOT, scrapePath)),
    restoredCount,
    unchangedCount,
    files: results
  };
}

// ── CLI ───────────────────────────────────────────────────────────────────────

async function main(): Promise<void> {
  const args    = process.argv.slice(2);
  const flags   = new Set(args.filter(a => a.startsWith("--")));
  const positional = args.filter(a => !a.startsWith("--"));

  function getFlag(name: string): string | undefined {
    const idx = args.indexOf(`--${name}`);
    return idx !== -1 && args[idx + 1] && !args[idx + 1].startsWith("--") ? args[idx + 1] : undefined;
  }

  const projectArg = getFlag("project");
  const pathArg    = getFlag("path");
  const restoreArg = getFlag("restore") || (flags.has("--restore") ? positional[0] : undefined);
  const labelArg   = getFlag("label");
  const dryRun     = flags.has("--dry-run");
  const isList     = flags.has("--list");
  const isSystem   = flags.has("--system");
  const isAll      = flags.has("--all");

  if (isList) {
    const entries = await listScrapes({ project: projectArg, pathArg, system: isSystem, all: isAll });
    if (!entries.length) { log("No scrapes found."); return; }
    log(`Found ${entries.length} scrape(s):\n`);
    for (const e of entries) {
      console.log(`  ${e.name}`);
      console.log(`    path:     ${e.path}`);
      console.log(`    serial:   ${e.serial}`);
      console.log(`    created: ${e.createdApprox}`);
      console.log();
    }
    return;
  }

  if (restoreArg) {
    log(`Restoring from: ${restoreArg}${dryRun ? " (dry run)" : ""}`);
    const result = await restoreFromScrapeFile(restoreArg, dryRun);
    log(`  ${result.restoredCount} file(s) restored, ${result.unchangedCount} unchanged`);
    for (const f of result.files) {
      if (f.action !== "unchanged") console.log(`  [${f.action}] ${f.filePath}`);
    }
    if (dryRun) log("Dry run — no files written.");
    return;
  }

  log(`Creating scrape${isSystem ? ` (system files)` : projectArg ? ` for project: ${projectArg}` : pathArg ? ` for path: ${pathArg}` : ` for active project`}...`);

  const result = await createScrape({ project: projectArg, pathArg, system: isSystem, label: labelArg });
  log(`Done. Output: ${result.scrapeFile}`);
}

if (require.main === module) {
  main().catch(err => { console.error("[scrape] Fatal:", err?.message || String(err)); process.exit(1); });
}