import { execFile } from "node:child_process"; import { copyFile, mkdir, mkdtemp, readFile, rename, rm, stat, utimes } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join, posix, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { parse } from "dotenv"; export interface SitemapSyncConfig { host: string; port: number; sourceDirectory: string; destinationDirectory: string; } export type SitemapTransfer = (name: string, destination: string) => Promise; const shardName = /^sitemap-[a-z0-9-]+\.xml$/; const execute = promisify(execFile); function validated(config: SitemapSyncConfig): SitemapSyncConfig { if (!/^(?:[A-Za-z_][A-Za-z0-9_.-]*@)?(?:[A-Za-z0-9][A-Za-z0-9.-]*|\[[A-Fa-f0-9:]+\])$/.test(config.host)) throw new Error("SITEMAP_SYNC_HOST must be an SSH host, optionally prefixed with a username"); if (!Number.isInteger(config.port) || config.port < 1 || config.port > 65535) throw new Error("SITEMAP_SYNC_PORT must be an integer between 1 and 65535"); if (!/^\/(?:[A-Za-z0-9_.-]+\/)*[A-Za-z0-9_.-]+\/?$/.test(config.sourceDirectory) || config.sourceDirectory.split("/").some((part) => part === "." || part === "..")) throw new Error("SITEMAP_SYNC_DIR must be an absolute remote directory without shell syntax or traversal"); if (!config.destinationDirectory || /[\0\r\n]/.test(config.destinationDirectory)) throw new Error("SITEMAP_DIR must identify a local directory"); const destinationDirectory = resolve(config.destinationDirectory); if (dirname(destinationDirectory) === destinationDirectory) throw new Error("SITEMAP_DIR cannot be a filesystem root"); return { ...config, sourceDirectory: posix.normalize(config.sourceDirectory), destinationDirectory }; } /** This explicit developer command always loads development configuration; it is never run by a request handler. */ export async function readSitemapSyncConfig(root = process.cwd(), environment: Record = process.env): Promise { if (environment.APP_ENV && environment.APP_ENV !== "development") throw new Error("sitemap:sync only supports APP_ENV=development; do not run inside a test or production service environment"); const settings = async (file: string) => { try { return parse(await readFile(join(root, file))); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return {}; throw error; } }; const inherited = Object.fromEntries(Object.entries(environment).filter((entry) => entry[1] !== undefined)); const values = { ...await settings("config/development.env"), ...await settings("config/development.local.env"), ...inherited }; if (values.APP_ENV && values.APP_ENV !== "development") throw new Error("sitemap:sync requires development configuration"); const port = (values.SITEMAP_SYNC_PORT || "22").trim(); if (!/^\d+$/.test(port)) throw new Error("SITEMAP_SYNC_PORT must be numeric"); return validated({ host: (values.SITEMAP_SYNC_HOST || "").trim(), port: Number(port), sourceDirectory: (values.SITEMAP_SYNC_DIR || "").trim(), destinationDirectory: resolve(root, values.SITEMAP_DIR?.trim() || "./template/pages/sitemaps"), }); } interface XMLNode { name: string; children: XMLNode[]; text: string } const xmlWhitespace = /^[ \t\r\n]*$/; const xmlName = "[A-Za-z_][A-Za-z0-9_.-]*(?::[A-Za-z_][A-Za-z0-9_.-]*)?"; function decodeXML(value: string): string { return value.replace(/&([^;]*);|&/g, (_all, entity: string | undefined) => { const named: Record = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'" }; if (entity && Object.hasOwn(named, entity)) return named[entity]!; const numeric = entity && /^(?:#\d+|#x[0-9a-fA-F]+)$/.test(entity) ? Number.parseInt(entity.slice(entity[1] === "x" ? 2 : 1), entity[1] === "x" ? 16 : 10) : NaN; if (!Number.isInteger(numeric) || numeric > 0x10ffff || (numeric >= 0xd800 && numeric <= 0xdfff) || (numeric < 0x20 && ![9, 10, 13].includes(numeric)) || [0xfffe, 0xffff].includes(numeric)) throw new Error("Invalid XML entity"); return String.fromCodePoint(numeric); }); } /** Validate well-formed sitemap XML without a network-capable parser or external entities. Bytes are never serialized again. */ function document(bytes: Uint8Array, file: string): XMLNode { try { const source = new TextDecoder("utf-8", { fatal: true }).decode(bytes).replace(/^\ufeff/, ""); if (/[\u0000-\u0008\u000b\u000c\u000e-\u001f\ufffe\uffff]/u.test(source) || / { if (!stack.length) { if (cdata || !xmlWhitespace.test(value)) throw new Error("XML text outside its root"); return; } if (!cdata && value.includes("]]>")) throw new Error("Invalid XML text"); stack[stack.length - 1]!.text += cdata ? value : decodeXML(value); }; while (position < source.length) { const start = source.indexOf("<", position); if (start < 0) { text(source.slice(position)); break; } text(source.slice(position, start)); if (source.startsWith("", start + 4); const contents = source.slice(start + 4, end); if (end < 0 || contents.includes("--") || contents.endsWith("-")) throw new Error("Invalid XML comment"); position = end + 3; continue; } if (source.startsWith("", start + 9); if (end < 0) throw new Error("Unclosed XML CDATA"); text(source.slice(start + 9, end), true); position = end + 3; continue; } if (source.startsWith("", start + 2); if (end < 0) throw new Error("Unclosed XML processing instruction"); const instruction = source.slice(start + 2, end); const target = new RegExp(`^(${xmlName})(?:[ \\t\\r\\n]+[\\s\\S]*)?$`).exec(instruction)?.[1]; if (!target) throw new Error("Invalid XML processing instruction"); if (/^xml$/i.test(target) && (start !== 0 || !/^xml\s+version\s*=\s*(["'])1\.0\1(?:\s+encoding\s*=\s*(["'])UTF-8\2)?(?:\s+standalone\s*=\s*(["'])(?:yes|no)\3)?\s*$/i.test(instruction) || target !== "xml")) throw new Error("Invalid XML declaration"); position = end + 2; continue; } let end = start + 1, quote = ""; for (; end < source.length; end++) { const character = source[end]!; if (quote) { if (character === quote) quote = ""; } else if (character === '"' || character === "'") quote = character; else if (character === ">") break; } if (end === source.length) throw new Error("Unclosed XML tag"); const tag = source.slice(start + 1, end); if (tag.startsWith("/")) { const match = new RegExp(`^/(${xmlName})[ \\t\\r\\n]*$`).exec(tag); if (!match || stack.pop()?.name !== match[1]) throw new Error("Mismatched XML closing tag"); } else { const selfClosing = tag.endsWith("/"), value = selfClosing ? tag.slice(0, -1) : tag; const name = new RegExp(`^(${xmlName})`).exec(value); if (!name) throw new Error("Invalid XML tag"); let attributes = value.slice(name[0].length); const seen = new Set(); while (!xmlWhitespace.test(attributes)) { const attribute = new RegExp(`^[ \\t\\r\\n]+(${xmlName})[ \\t\\r\\n]*=[ \\t\\r\\n]*(?:"([^"<]*)"|'([^'<]*)')`).exec(attributes); if (!attribute || seen.has(attribute[1]!)) throw new Error("Invalid XML attribute"); seen.add(attribute[1]!); decodeXML(attribute[2] ?? attribute[3] ?? ""); attributes = attributes.slice(attribute[0].length); } const node: XMLNode = { name: name[1]!, children: [], text: "" }; if (stack.length) stack[stack.length - 1]!.children.push(node); else { if (root) throw new Error("Multiple XML roots"); root = node; } if (!selfClosing) stack.push(node); } position = end + 1; } if (!root || stack.length) throw new Error("Incomplete XML document"); return root; } catch { throw new Error(`${file} is not valid UTF-8 sitemap XML`); } } const localName = (node: XMLNode) => node.name.split(":").at(-1); function location(item: XMLNode, file: string): URL { const locations = item.children.filter((child) => localName(child) === "loc"); if (locations.length !== 1 || locations[0]!.children.length || !xmlWhitespace.test(item.text)) throw new Error(`${file} contains an invalid location entry`); let url: URL; try { url = new URL(locations[0]!.text.trim()); } catch { throw new Error(`${file} contains an invalid location URL`); } if (!["http:", "https:"].includes(url.protocol) || url.username || url.password) throw new Error(`${file} contains an unsupported location URL`); return url; } function shardNames(bytes: Uint8Array): string[] { const root = document(bytes, "sitemap.xml"); if (localName(root) !== "sitemapindex" || !xmlWhitespace.test(root.text)) throw new Error("sitemap.xml must contain a sitemapindex"); const names = new Set(); for (const item of root.children) { if (localName(item) !== "sitemap") throw new Error("sitemap.xml contains an invalid sitemap entry"); const url = location(item, "sitemap.xml"); const name = /^\/sitemaps\/([^/]+)$/.exec(url.pathname)?.[1]; if (!name || !shardName.test(name) || !["http:", "https:"].includes(url.protocol) || url.username || url.password || url.search || url.hash) throw new Error("sitemap.xml contains an unsupported shard path"); names.add(name); } return [...names]; } function validateShard(bytes: Uint8Array, file: string): void { const root = document(bytes, file); if (localName(root) !== "urlset" || !xmlWhitespace.test(root.text)) throw new Error(`${file} must contain a urlset`); for (const item of root.children) { if (localName(item) !== "url") throw new Error(`${file} contains an invalid url entry`); location(item, file); } } function sshTransfer(config: SitemapSyncConfig): SitemapTransfer { return async (name, destination) => { if (name !== "sitemap.xml" && !shardName.test(name)) throw new Error("Invalid sitemap transfer filename"); try { await execute("scp", ["-p", "-q", "-P", String(config.port), "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes", "-o", "ConnectTimeout=15", "--", `${config.host}:${posix.join(config.sourceDirectory, name)}`, destination], { timeout: 120_000, maxBuffer: 1024 * 1024 }); } catch { throw new Error(`Unable to read remote ${name} over SSH`); } }; } /** Download and validate a complete generation before replacing any local XML; install its original index last. */ export async function syncSitemaps(settings: SitemapSyncConfig, transfer?: SitemapTransfer) { const config = validated(settings), copy = transfer || sshTransfer(config); const temporary = await mkdtemp(join(tmpdir(), "huakui-sitemap-sync-")); let installation: string | undefined; try { await copy("sitemap.xml", join(temporary, "sitemap.xml")); const index = await readFile(join(temporary, "sitemap.xml")), names = shardNames(index); for (const name of names) { await copy(name, join(temporary, name)); validateShard(await readFile(join(temporary, name)), name); } await copy("sitemap.xml", join(temporary, "index-check.xml")); if (!index.equals(await readFile(join(temporary, "index-check.xml")))) throw new Error("Remote sitemap index changed during synchronization; retry the command"); await mkdir(config.destinationDirectory, { recursive: true }); installation = await mkdtemp(join(config.destinationDirectory, ".sitemap-sync-")); const files = [...names, "sitemap.xml"]; for (const name of files) { const source = join(temporary, name), destination = join(installation, name), info = await stat(source); if (!info.isFile()) throw new Error(`${name} is not a regular file`); await copyFile(source, destination); await utimes(destination, info.atime, info.mtime); } for (const name of files) await rename(join(installation, name), join(config.destinationDirectory, name)); return { directory: config.destinationDirectory, files }; } finally { if (installation) await rm(installation, { recursive: true, force: true }); await rm(temporary, { recursive: true, force: true }); } } if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { try { if (process.argv.length > 2) throw new Error("Usage: npm run sitemap:sync (configure development.env, development.local.env or process environment)"); const result = await syncSitemaps(await readSitemapSyncConfig()); console.log(`Synchronized ${result.files.length} backend XML files to ${result.directory}; original bytes and modification times preserved.`); } catch (error) { console.error(error instanceof Error ? error.message : "Sitemap synchronization failed"); process.exitCode = 1; } }