web_huakui_web/scripts/sync-sitemap.ts

import { execFile } from "node:child_process";
import { copyFile, mkdir, mkdtemp, readFile, rename, rm, stat, utimes } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, join, posix, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { parse } from "dotenv";

export interface SitemapSyncConfig {
  host: string;
  port: number;
  sourceDirectory: string;
  destinationDirectory: string;
}
export type SitemapTransfer = (name: string, destination: string) => Promise<void>;
const shardName = /^sitemap-[a-z0-9-]+\.xml$/;
const execute = promisify(execFile);

function validated(config: SitemapSyncConfig): SitemapSyncConfig {
  if (!/^(?:[A-Za-z_][A-Za-z0-9_.-]*@)?(?:[A-Za-z0-9][A-Za-z0-9.-]*|\[[A-Fa-f0-9:]+\])$/.test(config.host))
    throw new Error("SITEMAP_SYNC_HOST must be an SSH host, optionally prefixed with a username");
  if (!Number.isInteger(config.port) || config.port < 1 || config.port > 65535)
    throw new Error("SITEMAP_SYNC_PORT must be an integer between 1 and 65535");
  if (!/^\/(?:[A-Za-z0-9_.-]+\/)*[A-Za-z0-9_.-]+\/?$/.test(config.sourceDirectory) || config.sourceDirectory.split("/").some((part) => part === "." || part === ".."))
    throw new Error("SITEMAP_SYNC_DIR must be an absolute remote directory without shell syntax or traversal");
  if (!config.destinationDirectory || /[\0\r\n]/.test(config.destinationDirectory))
    throw new Error("SITEMAP_DIR must identify a local directory");
  const destinationDirectory = resolve(config.destinationDirectory);
  if (dirname(destinationDirectory) === destinationDirectory)
    throw new Error("SITEMAP_DIR cannot be a filesystem root");
  return { ...config, sourceDirectory: posix.normalize(config.sourceDirectory), destinationDirectory };
}

/** This explicit developer command always loads development configuration; it is never run by a request handler. */
export async function readSitemapSyncConfig(root = process.cwd(), environment: Record<string, string | undefined> = process.env): Promise<SitemapSyncConfig> {
  if (environment.APP_ENV && environment.APP_ENV !== "development")
    throw new Error("sitemap:sync only supports APP_ENV=development; do not run inside a test or production service environment");
  const settings = async (file: string) => {
    try { return parse(await readFile(join(root, file))); }
    catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return {}; throw error; }
  };
  const inherited = Object.fromEntries(Object.entries(environment).filter((entry) => entry[1] !== undefined));
  const values = { ...await settings("config/development.env"), ...await settings("config/development.local.env"), ...inherited };
  if (values.APP_ENV && values.APP_ENV !== "development")
    throw new Error("sitemap:sync requires development configuration");
  const port = (values.SITEMAP_SYNC_PORT || "22").trim();
  if (!/^\d+$/.test(port)) throw new Error("SITEMAP_SYNC_PORT must be numeric");
  return validated({
    host: (values.SITEMAP_SYNC_HOST || "").trim(),
    port: Number(port),
    sourceDirectory: (values.SITEMAP_SYNC_DIR || "").trim(),
    destinationDirectory: resolve(root, values.SITEMAP_DIR?.trim() || "./template/pages/sitemaps"),
  });
}

interface XMLNode { name: string; children: XMLNode[]; text: string }
const xmlWhitespace = /^[ \t\r\n]*$/;
const xmlName = "[A-Za-z_][A-Za-z0-9_.-]*(?::[A-Za-z_][A-Za-z0-9_.-]*)?";
function decodeXML(value: string): string {
  return value.replace(/&([^;]*);|&/g, (_all, entity: string | undefined) => {
    const named: Record<string, string> = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'" };
    if (entity && Object.hasOwn(named, entity)) return named[entity]!;
    const numeric = entity && /^(?:#\d+|#x[0-9a-fA-F]+)$/.test(entity)
      ? Number.parseInt(entity.slice(entity[1] === "x" ? 2 : 1), entity[1] === "x" ? 16 : 10) : NaN;
    if (!Number.isInteger(numeric) || numeric > 0x10ffff || (numeric >= 0xd800 && numeric <= 0xdfff) || (numeric < 0x20 && ![9, 10, 13].includes(numeric)) || [0xfffe, 0xffff].includes(numeric))
      throw new Error("Invalid XML entity");
    return String.fromCodePoint(numeric);
  });
}

/** Validate well-formed sitemap XML without a network-capable parser or external entities. Bytes are never serialized again. */
function document(bytes: Uint8Array, file: string): XMLNode {
  try {
    const source = new TextDecoder("utf-8", { fatal: true }).decode(bytes).replace(/^\ufeff/, "");
    if (/[\u0000-\u0008\u000b\u000c\u000e-\u001f\ufffe\uffff]/u.test(source) || /<!DOCTYPE/i.test(source)) throw new Error("Unsupported XML declaration");
    const stack: XMLNode[] = [];
    let root: XMLNode | undefined, position = 0;
    const text = (value: string, cdata = false) => {
      if (!stack.length) { if (cdata || !xmlWhitespace.test(value)) throw new Error("XML text outside its root"); return; }
      if (!cdata && value.includes("]]>")) throw new Error("Invalid XML text");
      stack[stack.length - 1]!.text += cdata ? value : decodeXML(value);
    };
    while (position < source.length) {
      const start = source.indexOf("<", position);
      if (start < 0) { text(source.slice(position)); break; }
      text(source.slice(position, start));
      if (source.startsWith("<!--", start)) {
        const end = source.indexOf("-->", start + 4);
        const contents = source.slice(start + 4, end);
        if (end < 0 || contents.includes("--") || contents.endsWith("-")) throw new Error("Invalid XML comment");
        position = end + 3; continue;
      }
      if (source.startsWith("<![CDATA[", start)) {
        const end = source.indexOf("]]>", start + 9);
        if (end < 0) throw new Error("Unclosed XML CDATA");
        text(source.slice(start + 9, end), true); position = end + 3; continue;
      }
      if (source.startsWith("<?", start)) {
        const end = source.indexOf("?>", start + 2);
        if (end < 0) throw new Error("Unclosed XML processing instruction");
        const instruction = source.slice(start + 2, end);
        const target = new RegExp(`^(${xmlName})(?:[ \\t\\r\\n]+[\\s\\S]*)?$`).exec(instruction)?.[1];
        if (!target) throw new Error("Invalid XML processing instruction");
        if (/^xml$/i.test(target) && (start !== 0 || !/^xml\s+version\s*=\s*(["'])1\.0\1(?:\s+encoding\s*=\s*(["'])UTF-8\2)?(?:\s+standalone\s*=\s*(["'])(?:yes|no)\3)?\s*$/i.test(instruction) || target !== "xml"))
          throw new Error("Invalid XML declaration");
        position = end + 2; continue;
      }
      let end = start + 1, quote = "";
      for (; end < source.length; end++) {
        const character = source[end]!;
        if (quote) { if (character === quote) quote = ""; }
        else if (character === '"' || character === "'") quote = character;
        else if (character === ">") break;
      }
      if (end === source.length) throw new Error("Unclosed XML tag");
      const tag = source.slice(start + 1, end);
      if (tag.startsWith("/")) {
        const match = new RegExp(`^/(${xmlName})[ \\t\\r\\n]*$`).exec(tag);
        if (!match || stack.pop()?.name !== match[1]) throw new Error("Mismatched XML closing tag");
      } else {
        const selfClosing = tag.endsWith("/"), value = selfClosing ? tag.slice(0, -1) : tag;
        const name = new RegExp(`^(${xmlName})`).exec(value);
        if (!name) throw new Error("Invalid XML tag");
        let attributes = value.slice(name[0].length);
        const seen = new Set<string>();
        while (!xmlWhitespace.test(attributes)) {
          const attribute = new RegExp(`^[ \\t\\r\\n]+(${xmlName})[ \\t\\r\\n]*=[ \\t\\r\\n]*(?:"([^"<]*)"|'([^'<]*)')`).exec(attributes);
          if (!attribute || seen.has(attribute[1]!)) throw new Error("Invalid XML attribute");
          seen.add(attribute[1]!); decodeXML(attribute[2] ?? attribute[3] ?? "");
          attributes = attributes.slice(attribute[0].length);
        }
        const node: XMLNode = { name: name[1]!, children: [], text: "" };
        if (stack.length) stack[stack.length - 1]!.children.push(node);
        else { if (root) throw new Error("Multiple XML roots"); root = node; }
        if (!selfClosing) stack.push(node);
      }
      position = end + 1;
    }
    if (!root || stack.length) throw new Error("Incomplete XML document");
    return root;
  } catch { throw new Error(`${file} is not valid UTF-8 sitemap XML`); }
}
const localName = (node: XMLNode) => node.name.split(":").at(-1);
function location(item: XMLNode, file: string): URL {
  const locations = item.children.filter((child) => localName(child) === "loc");
  if (locations.length !== 1 || locations[0]!.children.length || !xmlWhitespace.test(item.text))
    throw new Error(`${file} contains an invalid location entry`);
  let url: URL;
  try { url = new URL(locations[0]!.text.trim()); } catch { throw new Error(`${file} contains an invalid location URL`); }
  if (!["http:", "https:"].includes(url.protocol) || url.username || url.password)
    throw new Error(`${file} contains an unsupported location URL`);
  return url;
}
function shardNames(bytes: Uint8Array): string[] {
  const root = document(bytes, "sitemap.xml");
  if (localName(root) !== "sitemapindex" || !xmlWhitespace.test(root.text)) throw new Error("sitemap.xml must contain a sitemapindex");
  const names = new Set<string>();
  for (const item of root.children) {
    if (localName(item) !== "sitemap") throw new Error("sitemap.xml contains an invalid sitemap entry");
    const url = location(item, "sitemap.xml");
    const name = /^\/sitemaps\/([^/]+)$/.exec(url.pathname)?.[1];
    if (!name || !shardName.test(name) || !["http:", "https:"].includes(url.protocol) || url.username || url.password || url.search || url.hash)
      throw new Error("sitemap.xml contains an unsupported shard path");
    names.add(name);
  }
  return [...names];
}
function validateShard(bytes: Uint8Array, file: string): void {
  const root = document(bytes, file);
  if (localName(root) !== "urlset" || !xmlWhitespace.test(root.text)) throw new Error(`${file} must contain a urlset`);
  for (const item of root.children) {
    if (localName(item) !== "url") throw new Error(`${file} contains an invalid url entry`);
    location(item, file);
  }
}

function sshTransfer(config: SitemapSyncConfig): SitemapTransfer {
  return async (name, destination) => {
    if (name !== "sitemap.xml" && !shardName.test(name)) throw new Error("Invalid sitemap transfer filename");
    try {
      await execute("scp", ["-p", "-q", "-P", String(config.port), "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes", "-o", "ConnectTimeout=15", "--", `${config.host}:${posix.join(config.sourceDirectory, name)}`, destination], { timeout: 120_000, maxBuffer: 1024 * 1024 });
    } catch { throw new Error(`Unable to read remote ${name} over SSH`); }
  };
}

/** Download and validate a complete generation before replacing any local XML; install its original index last. */
export async function syncSitemaps(settings: SitemapSyncConfig, transfer?: SitemapTransfer) {
  const config = validated(settings), copy = transfer || sshTransfer(config);
  const temporary = await mkdtemp(join(tmpdir(), "huakui-sitemap-sync-"));
  let installation: string | undefined;
  try {
    await copy("sitemap.xml", join(temporary, "sitemap.xml"));
    const index = await readFile(join(temporary, "sitemap.xml")), names = shardNames(index);
    for (const name of names) {
      await copy(name, join(temporary, name));
      validateShard(await readFile(join(temporary, name)), name);
    }
    await copy("sitemap.xml", join(temporary, "index-check.xml"));
    if (!index.equals(await readFile(join(temporary, "index-check.xml")))) throw new Error("Remote sitemap index changed during synchronization; retry the command");
    await mkdir(config.destinationDirectory, { recursive: true });
    installation = await mkdtemp(join(config.destinationDirectory, ".sitemap-sync-"));
    const files = [...names, "sitemap.xml"];
    for (const name of files) {
      const source = join(temporary, name), destination = join(installation, name), info = await stat(source);
      if (!info.isFile()) throw new Error(`${name} is not a regular file`);
      await copyFile(source, destination);
      await utimes(destination, info.atime, info.mtime);
    }
    for (const name of files) await rename(join(installation, name), join(config.destinationDirectory, name));
    return { directory: config.destinationDirectory, files };
  } finally {
    if (installation) await rm(installation, { recursive: true, force: true });
    await rm(temporary, { recursive: true, force: true });
  }
}

if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
  try {
    if (process.argv.length > 2) throw new Error("Usage: npm run sitemap:sync (configure development.env, development.local.env or process environment)");
    const result = await syncSitemaps(await readSitemapSyncConfig());
    console.log(`Synchronized ${result.files.length} backend XML files to ${result.directory}; original bytes and modification times preserved.`);
  } catch (error) { console.error(error instanceof Error ? error.message : "Sitemap synchronization failed"); process.exitCode = 1; }
}