import assert from 'node:assert/strict'; import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import test from 'node:test'; import { createContext } from '../src/server/context'; import { resolveSiteURL, usableSiteURL } from '../src/server/seo/shared'; import { BUNDLED_ROBOTS_TXT, BUILTIN_ROBOTS_TXT, ROBOTS_TEMPLATE_PATHS, buildLLMSTXT, buildRobotsTXT, loadDiscoveryConfig, readRobotsTXT, } from '../src/server/seo/discovery-text'; const site = 'https://canonical.drama.test'; test('robots follows the four original disk paths, skips all read errors/empty text, then uses embedded and builtin text', async () => { const cwd = await mkdtemp(path.join(os.tmpdir(), 'huakui-go-robots-')); const files = ROBOTS_TEMPLATE_PATHS.map(relative => path.join(cwd, relative)); try { await Promise.all(files.map(file => mkdir(path.dirname(file), { recursive: true }))); await mkdir(files[0]!); // os.ReadFile(directory) fails; the next candidate must still be considered. await writeFile(files[1]!, ' \r\n\t\u0085\u00a0'); await writeFile(files[2]!, ' User-agent: *\r\n# third template\r\nSitemap: /old.xml\r\nHost: intentional.drama.test\r\n'); await writeFile(files[3]!, 'User-agent: *\n# fourth template'); await mkdir(path.join(cwd, 'config/seo'), { recursive: true }); await writeFile(path.join(cwd, 'config/seo/robots.txt'), 'Disallow: /'); assert.equal(await readRobotsTXT(site, { cwd }), 'User-agent: *\n# third template\nSitemap: ' + site + '/sitemap.xml\nHost: intentional.drama.test\n'); await rm(files[0]!, { recursive: true }); await writeFile(files[0]!, 'User-agent: *\n# first template'); assert.equal(await readRobotsTXT(site, { cwd }), 'User-agent: *\n# first template\n\nSitemap: ' + site + '/sitemap.xml\nHost: canonical.drama.test\n'); await Promise.all([files[0]!, files[2]!, files[3]!].map(file => rm(file))); assert.equal(await readRobotsTXT(site, { cwd }), buildRobotsTXT(site, BUNDLED_ROBOTS_TXT)); assert.equal(await readRobotsTXT(site, { cwd, bundledText: null }), buildRobotsTXT(site, BUILTIN_ROBOTS_TXT)); assert.equal(await readRobotsTXT(site, { cwd, bundledText: '\r\n\t' }), buildRobotsTXT(site, BUILTIN_ROBOTS_TXT)); } finally { await rm(cwd, { recursive: true, force: true }); } }); test('robots preserves Go Sitemap/Host case, comments, empty-origin and final-newline behavior', () => { assert.equal(buildRobotsTXT(site, 'User-agent: *\r\nSitemap: /a\r\nSitemap: /b\r\nHost: keep.drama.test\r\n'), 'User-agent: *\nSitemap: ' + site + '/sitemap.xml\nSitemap: ' + site + '/sitemap.xml\nHost: keep.drama.test\n'); assert.equal(buildRobotsTXT('', 'User-agent: *\nSitemap: /old'), 'User-agent: *\nSitemap: /sitemap.xml\n'); assert.equal(buildRobotsTXT(site, 'User-agent: *\n# Sitemap: managed separately\n# Host: managed separately'), 'User-agent: *\n# Sitemap: managed separately\n# Host: managed separately\n'); assert.equal(buildRobotsTXT(site, 'User-agent: *\nsitemap: /lower\nhost: lower.drama.test'), 'User-agent: *\nsitemap: /lower\nhost: lower.drama.test\n\nSitemap: ' + site + '/sitemap.xml\nHost: canonical.drama.test\n'); assert.equal(buildRobotsTXT(site, '\u0085\u00a0User-agent: *\u0085'), 'User-agent: *\n\nSitemap: ' + site + '/sitemap.xml\nHost: canonical.drama.test\n'); assert.ok(buildRobotsTXT(site, '\ufeffUser-agent: *').startsWith('\ufeffUser-agent: *'), 'Go TrimSpace preserves BOM'); assert.equal(buildRobotsTXT('https://Case.drama.test:443', 'User-agent: *'), 'User-agent: *\n\nSitemap: https://Case.drama.test:443/sitemap.xml\nHost: Case.drama.test:443\n'); }); test('llms restores upstream SEO copy and Go Unicode Fields while keeping the approved brand and public links', () => { const text = buildLLMSTXT(' ' + site + '/// ', { SeoInfo: { WebSiteName: '\u0085黄果短剧\t 官方\n站点\u00a0', Description: ' 最新\n\t短剧\u0085更新 ' } }); assert.equal(text, '# 花魁短剧 官方 站点\n\n> 最新 短剧 更新\n\n## 核心页面\n\n' + '- [首页](' + site + '/)\n' + '- [短剧列表](' + site + '/dramas/)\n' + '- [剧集](' + site + '/series/)\n' + '- [热门推荐](' + site + '/featured/)\n' + '- [今日更新](' + site + '/latest/)\n' + '- [专题](' + site + '/collections/)\n' + '\n## 站点发现\n\n- [Sitemap](' + site + '/sitemap.xml)\n- [Robots](' + site + '/robots.txt)\n'); assert.match(buildLLMSTXT('', {}), /^# 花魁短剧\n\n> 汇集热门短剧/); assert.ok(buildLLMSTXT(site, { seoInfo: { webSiteName: '\ufeff花魁短剧' } }).startsWith('# \ufeff花魁短剧')); }); test('origin resolution matches Go permanent-site priority and request fallback without new deployment overrides', () => { const original = process.env.PUBLIC_SITE_URL; process.env.PUBLIC_SITE_URL = 'https://must-not-override.drama.test'; try { const ctx = createContext(new Request('http://internal:26935/robots.txt', { headers: { 'X-Forwarded-Proto': ' https, http ', 'X-Forwarded-Host': ' edge.drama.test, private.internal ', Host: 'request.drama.test', Forwarded: 'host=ignored.drama.test;proto=http', 'CF-Visitor': '{"scheme":"http"}' } })); assert.equal(resolveSiteURL(ctx, { foreverUrlAddr: site + '/path?discard=1' }), site); assert.equal(resolveSiteURL(ctx, { foreverUrlAddr: 'https://example.com' }), 'https://edge.drama.test'); assert.equal(resolveSiteURL(createContext(new Request('http://10.0.0.4:26935/robots.txt'))), 'http://10.0.0.4:26935'); assert.equal(resolveSiteURL(createContext(new Request('https://request.drama.test/robots.txt', { headers: { Forwarded: 'host=ignored.drama.test;proto=http', 'CF-Visitor': '{"scheme":"http"}' } }))), 'https://request.drama.test'); assert.equal(resolveSiteURL(createContext(new Request('http://request.drama.test/robots.txt', { headers: { 'X-Forwarded-Host': ', later.drama.test' } }))), ''); assert.equal(resolveSiteURL(createContext(new Request('https://request.drama.test/robots.txt', { headers: { 'X-Forwarded-Proto': ', https' } }))), '://request.drama.test'); assert.equal(usableSiteURL(' HTTPS://user:password@Case.drama.test:443/path?x=1#part '), 'https://Case.drama.test:443'); assert.equal(usableSiteURL('canonical.drama.test/path'), site); assert.equal(usableSiteURL('https://foo\n.drama.test'), ''); for (const value of ['https://a%25.drama.test', 'https://[fe80::1%25eth0]']) { assert.equal(usableSiteURL(value), '', value); assert.equal(resolveSiteURL(ctx, { foreverUrlAddr: value }), 'https://edge.drama.test', value); } assert.equal(usableSiteURL('https://a%2525.drama.test/path'), 'https://a%25.drama.test'); assert.equal(usableSiteURL('https://[fe80::1%2525eth0]:443/path'), 'https://[fe80::1%25eth0]:443'); for (const value of ['https://localhost', 'http://127.0.0.1', 'https://example.com', 'https://placeholder.example.com', 'https://placeholder.local', 'javascript:alert(1)']) assert.equal(usableSiteURL(value), '', value); } finally { if (original === undefined) delete process.env.PUBLIC_SITE_URL; else process.env.PUBLIC_SITE_URL = original; } }); test('public discovery config shares the 60-second success cache, stale fallback and concurrent refresh without visitor cookies', async () => { const originalFetch = globalThis.fetch; const originalURL = process.env.UPSTREAM_API_URL; const originalNow = Date.now; process.env.UPSTREAM_API_URL = 'https://go-discovery-cache.drama.test'; let now = originalNow(), calls = 0, fail = false; Date.now = () => now; globalThis.fetch = async (url, init) => { calls++; assert.equal(String(url), process.env.UPSTREAM_API_URL + '/api/app/config/info'); const headers = new Headers(init?.headers); assert.equal(headers.has('authorization'), false); assert.equal(headers.has('cookie'), false); if (fail) return new Response('offline', { status: 503 }); return Response.json({ code: 200, data: { config: { foreverUrlAddr: 'https://upstream.drama.test' }, seoInfo: { webSiteName: '黄果短剧 官方', description: '后台\n动态描述' } } }, { headers: { 'Set-Cookie': 'upstream-private=must-not-return; Path=/' } }); }; const context = () => createContext(new Request(site + '/robots.txt', { headers: { Cookie: 'token=registered-private', Authorization: 'private-session' } })); try { const visitors = [context(), context(), context()]; const configs = await Promise.all(visitors.map(loadDiscoveryConfig)); assert.equal(calls, 1); assert.equal(resolveSiteURL(visitors[0]!, configs[0]), 'https://upstream.drama.test'); assert.match(buildLLMSTXT(site, configs[0]), /^# 花魁短剧 官方\n\n> 后台 动态描述/); assert.ok(visitors.every(ctx => ctx.responseHeaders.getSetCookie().length === 0)); await loadDiscoveryConfig(context()); assert.equal(calls, 1, 'successful config remains cached for 60 seconds'); now += 60001; fail = true; const staleContext = context(); const stale = await loadDiscoveryConfig(staleContext); assert.equal(calls, 2); assert.equal(stale.foreverUrlAddr, 'https://upstream.drama.test'); assert.match(buildLLMSTXT(site, stale), /后台 动态描述/); assert.deepEqual(staleContext.responseHeaders.getSetCookie(), []); process.env.UPSTREAM_API_URL = 'https://go-discovery-cold-offline.drama.test'; const empty = await loadDiscoveryConfig(context()); assert.equal(calls, 3); assert.match(buildLLMSTXT('', empty), /^# 花魁短剧\n\n> 汇集热门短剧/); } finally { globalThis.fetch = originalFetch; Date.now = originalNow; if (originalURL === undefined) delete process.env.UPSTREAM_API_URL; else process.env.UPSTREAM_API_URL = originalURL; } });