web_huakui_web/tests/discovery-text.test.ts

import assert from 'node:assert/strict';
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import { createContext } from '../src/server/context';
import { resolveSiteURL, usableSiteURL } from '../src/server/seo/shared';
import {
  BUNDLED_ROBOTS_TXT, BUILTIN_ROBOTS_TXT, ROBOTS_TEMPLATE_PATHS,
  buildLLMSTXT, buildRobotsTXT, loadDiscoveryConfig, readRobotsTXT,
} from '../src/server/seo/discovery-text';

const site = 'https://canonical.drama.test';

test('robots follows the four original disk paths, skips all read errors/empty text, then uses embedded and builtin text', async () => {
  const cwd = await mkdtemp(path.join(os.tmpdir(), 'huakui-go-robots-'));
  const files = ROBOTS_TEMPLATE_PATHS.map(relative => path.join(cwd, relative));
  try {
    await Promise.all(files.map(file => mkdir(path.dirname(file), { recursive: true })));
    await mkdir(files[0]!); // os.ReadFile(directory) fails; the next candidate must still be considered.
    await writeFile(files[1]!, ' \r\n\t\u0085\u00a0');
    await writeFile(files[2]!, ' User-agent: *\r\n# third template\r\nSitemap: /old.xml\r\nHost: intentional.drama.test\r\n');
    await writeFile(files[3]!, 'User-agent: *\n# fourth template');
    await mkdir(path.join(cwd, 'config/seo'), { recursive: true });
    await writeFile(path.join(cwd, 'config/seo/robots.txt'), 'Disallow: /');
    assert.equal(await readRobotsTXT(site, { cwd }), 'User-agent: *\n# third template\nSitemap: ' + site + '/sitemap.xml\nHost: intentional.drama.test\n');
    await rm(files[0]!, { recursive: true });
    await writeFile(files[0]!, 'User-agent: *\n# first template');
    assert.equal(await readRobotsTXT(site, { cwd }), 'User-agent: *\n# first template\n\nSitemap: ' + site + '/sitemap.xml\nHost: canonical.drama.test\n');
    await Promise.all([files[0]!, files[2]!, files[3]!].map(file => rm(file)));
    assert.equal(await readRobotsTXT(site, { cwd }), buildRobotsTXT(site, BUNDLED_ROBOTS_TXT));
    assert.equal(await readRobotsTXT(site, { cwd, bundledText: null }), buildRobotsTXT(site, BUILTIN_ROBOTS_TXT));
    assert.equal(await readRobotsTXT(site, { cwd, bundledText: '\r\n\t' }), buildRobotsTXT(site, BUILTIN_ROBOTS_TXT));
  } finally { await rm(cwd, { recursive: true, force: true }); }
});

test('robots preserves Go Sitemap/Host case, comments, empty-origin and final-newline behavior', () => {
  assert.equal(buildRobotsTXT(site, 'User-agent: *\r\nSitemap: /a\r\nSitemap: /b\r\nHost: keep.drama.test\r\n'),
    'User-agent: *\nSitemap: ' + site + '/sitemap.xml\nSitemap: ' + site + '/sitemap.xml\nHost: keep.drama.test\n');
  assert.equal(buildRobotsTXT('', 'User-agent: *\nSitemap: /old'), 'User-agent: *\nSitemap: /sitemap.xml\n');
  assert.equal(buildRobotsTXT(site, 'User-agent: *\n# Sitemap: managed separately\n# Host: managed separately'),
    'User-agent: *\n# Sitemap: managed separately\n# Host: managed separately\n');
  assert.equal(buildRobotsTXT(site, 'User-agent: *\nsitemap: /lower\nhost: lower.drama.test'),
    'User-agent: *\nsitemap: /lower\nhost: lower.drama.test\n\nSitemap: ' + site + '/sitemap.xml\nHost: canonical.drama.test\n');
  assert.equal(buildRobotsTXT(site, '\u0085\u00a0User-agent: *\u0085'),
    'User-agent: *\n\nSitemap: ' + site + '/sitemap.xml\nHost: canonical.drama.test\n');
  assert.ok(buildRobotsTXT(site, '\ufeffUser-agent: *').startsWith('\ufeffUser-agent: *'), 'Go TrimSpace preserves BOM');
  assert.equal(buildRobotsTXT('https://Case.drama.test:443', 'User-agent: *'),
    'User-agent: *\n\nSitemap: https://Case.drama.test:443/sitemap.xml\nHost: Case.drama.test:443\n');
});

test('llms restores upstream SEO copy and Go Unicode Fields while keeping the approved brand and public links', () => {
  const text = buildLLMSTXT(' ' + site + '/// ', { SeoInfo: { WebSiteName: '\u0085黄果短剧\t 官方\n站点\u00a0', Description: ' 最新\n\t短剧\u0085更新 ' } });
  assert.equal(text, '# 花魁短剧 官方 站点\n\n> 最新 短剧 更新\n\n## 核心页面\n\n' +
    '- [首页](' + site + '/)\n' +
    '- [短剧列表](' + site + '/dramas/)\n' +
    '- [剧集](' + site + '/series/)\n' +
    '- [热门推荐](' + site + '/featured/)\n' +
    '- [今日更新](' + site + '/latest/)\n' +
    '- [专题](' + site + '/collections/)\n' +
    '\n## 站点发现\n\n- [Sitemap](' + site + '/sitemap.xml)\n- [Robots](' + site + '/robots.txt)\n');
  assert.match(buildLLMSTXT('', {}), /^# 花魁短剧\n\n> 汇集热门短剧/);
  assert.ok(buildLLMSTXT(site, { seoInfo: { webSiteName: '\ufeff花魁短剧' } }).startsWith('# \ufeff花魁短剧'));
});

test('origin resolution matches Go permanent-site priority and request fallback without new deployment overrides', () => {
  const original = process.env.PUBLIC_SITE_URL;
  process.env.PUBLIC_SITE_URL = 'https://must-not-override.drama.test';
  try {
    const ctx = createContext(new Request('http://internal:26935/robots.txt', { headers: { 'X-Forwarded-Proto': ' https, http ', 'X-Forwarded-Host': ' edge.drama.test, private.internal ', Host: 'request.drama.test', Forwarded: 'host=ignored.drama.test;proto=http', 'CF-Visitor': '{"scheme":"http"}' } }));
    assert.equal(resolveSiteURL(ctx, { foreverUrlAddr: site + '/path?discard=1' }), site);
    assert.equal(resolveSiteURL(ctx, { foreverUrlAddr: 'https://example.com' }), 'https://edge.drama.test');
    assert.equal(resolveSiteURL(createContext(new Request('http://10.0.0.4:26935/robots.txt'))), 'http://10.0.0.4:26935');
    assert.equal(resolveSiteURL(createContext(new Request('https://request.drama.test/robots.txt', { headers: { Forwarded: 'host=ignored.drama.test;proto=http', 'CF-Visitor': '{"scheme":"http"}' } }))), 'https://request.drama.test');
    assert.equal(resolveSiteURL(createContext(new Request('http://request.drama.test/robots.txt', { headers: { 'X-Forwarded-Host': ', later.drama.test' } }))), '');
    assert.equal(resolveSiteURL(createContext(new Request('https://request.drama.test/robots.txt', { headers: { 'X-Forwarded-Proto': ', https' } }))), '://request.drama.test');
    assert.equal(usableSiteURL(' HTTPS://user:password@Case.drama.test:443/path?x=1#part '), 'https://Case.drama.test:443');
    assert.equal(usableSiteURL('canonical.drama.test/path'), site);
    assert.equal(usableSiteURL('https://foo\n.drama.test'), '');
    for (const value of ['https://a%25.drama.test', 'https://[fe80::1%25eth0]']) {
      assert.equal(usableSiteURL(value), '', value);
      assert.equal(resolveSiteURL(ctx, { foreverUrlAddr: value }), 'https://edge.drama.test', value);
    }
    assert.equal(usableSiteURL('https://a%2525.drama.test/path'), 'https://a%25.drama.test');
    assert.equal(usableSiteURL('https://[fe80::1%2525eth0]:443/path'), 'https://[fe80::1%25eth0]:443');
    for (const value of ['https://localhost', 'http://127.0.0.1', 'https://example.com', 'https://placeholder.example.com', 'https://placeholder.local', 'javascript:alert(1)']) assert.equal(usableSiteURL(value), '', value);
  } finally {
    if (original === undefined) delete process.env.PUBLIC_SITE_URL; else process.env.PUBLIC_SITE_URL = original;
  }
});

test('public discovery config shares the 60-second success cache, stale fallback and concurrent refresh without visitor cookies', async () => {
  const originalFetch = globalThis.fetch;
  const originalURL = process.env.UPSTREAM_API_URL;
  const originalNow = Date.now;
  process.env.UPSTREAM_API_URL = 'https://go-discovery-cache.drama.test';
  let now = originalNow(), calls = 0, fail = false;
  Date.now = () => now;
  globalThis.fetch = async (url, init) => {
    calls++;
    assert.equal(String(url), process.env.UPSTREAM_API_URL + '/api/app/config/info');
    const headers = new Headers(init?.headers);
    assert.equal(headers.has('authorization'), false);
    assert.equal(headers.has('cookie'), false);
    if (fail) return new Response('offline', { status: 503 });
    return Response.json({ code: 200, data: { config: { foreverUrlAddr: 'https://upstream.drama.test' }, seoInfo: { webSiteName: '黄果短剧 官方', description: '后台\n动态描述' } } }, { headers: { 'Set-Cookie': 'upstream-private=must-not-return; Path=/' } });
  };
  const context = () => createContext(new Request(site + '/robots.txt', { headers: { Cookie: 'token=registered-private', Authorization: 'private-session' } }));
  try {
    const visitors = [context(), context(), context()];
    const configs = await Promise.all(visitors.map(loadDiscoveryConfig));
    assert.equal(calls, 1);
    assert.equal(resolveSiteURL(visitors[0]!, configs[0]), 'https://upstream.drama.test');
    assert.match(buildLLMSTXT(site, configs[0]), /^# 花魁短剧 官方\n\n> 后台 动态描述/);
    assert.ok(visitors.every(ctx => ctx.responseHeaders.getSetCookie().length === 0));
    await loadDiscoveryConfig(context());
    assert.equal(calls, 1, 'successful config remains cached for 60 seconds');
    now += 60001;
    fail = true;
    const staleContext = context();
    const stale = await loadDiscoveryConfig(staleContext);
    assert.equal(calls, 2);
    assert.equal(stale.foreverUrlAddr, 'https://upstream.drama.test');
    assert.match(buildLLMSTXT(site, stale), /后台 动态描述/);
    assert.deepEqual(staleContext.responseHeaders.getSetCookie(), []);
    process.env.UPSTREAM_API_URL = 'https://go-discovery-cold-offline.drama.test';
    const empty = await loadDiscoveryConfig(context());
    assert.equal(calls, 3);
    assert.match(buildLLMSTXT('', empty), /^# 花魁短剧\n\n> 汇集热门短剧/);
  } finally {
    globalThis.fetch = originalFetch;
    Date.now = originalNow;
    if (originalURL === undefined) delete process.env.UPSTREAM_API_URL; else process.env.UPSTREAM_API_URL = originalURL;
  }
});