117 lines
4.3 KiB
JavaScript
117 lines
4.3 KiB
JavaScript
#!/usr/bin/env node
|
||
'use strict';
|
||
|
||
// 抓取官网 sitemap.xml（www/us），把全量 URL 落到 bridge 库 bridge_site_map_urls。
|
||
// 数据只在服务端匹配，不进 LLM/向量库；商品搜索无结果时返回真实商品页链接。
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
const { Client } = require('../bridge/node_modules/pg');
|
||
|
||
const ROOT = path.resolve(__dirname, '..');
|
||
const env = {};
|
||
for (const line of fs.readFileSync(path.join(ROOT, '.env'), 'utf8').split('\n')) {
|
||
if (!line.includes('=') || line.trim().startsWith('#')) continue;
|
||
const i = line.indexOf('=');
|
||
env[line.slice(0, i).trim()] = line.slice(i + 1).trim();
|
||
}
|
||
const password = encodeURIComponent(env.POSTGRES_PASSWORD || '');
|
||
const PG_URL = process.env.SITEMAP_PG_URL || `postgres://crmnew:${password}@127.0.0.1:5440/bridge`;
|
||
|
||
const SITES = {
|
||
www: { base: 'https://www.yehwang.com', sitemap: 'https://www.yehwang.com/sitemap.xml' },
|
||
us: { base: 'https://us.yehwang.com', sitemap: 'https://us.yehwang.com/sitemap.xml' },
|
||
};
|
||
|
||
function tokensFor(pathPart) {
|
||
return pathPart
|
||
.replace(/\.html$/, '')
|
||
.split('/')
|
||
.filter(Boolean)
|
||
.map((seg) => seg.replace(/-?\d+$/, '').split('-').filter((w) => w && w.length > 1).join(' '))
|
||
.filter(Boolean)
|
||
.join(' ')
|
||
.toLowerCase();
|
||
}
|
||
|
||
async function fetchSitemap(url) {
|
||
const res = await fetch(url, {
|
||
headers: { 'User-Agent': 'Mozilla/5.0 (compatible; YehwangSiteMapBot/1.0)' },
|
||
signal: AbortSignal.timeout(90000),
|
||
});
|
||
if (!res.ok) throw new Error(`sitemap HTTP ${res.status}: ${url}`);
|
||
const xml = await res.text();
|
||
return [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1].trim()).filter(Boolean);
|
||
}
|
||
|
||
async function buildSite(client, name) {
|
||
const cfg = SITES[name];
|
||
const urls = await fetchSitemap(cfg.sitemap);
|
||
const rows = [];
|
||
const seenPaths = new Set();
|
||
for (const url of urls) {
|
||
if (!url.startsWith(cfg.base)) continue;
|
||
const pathPart = url.slice(cfg.base.length).split('?')[0];
|
||
if (!pathPart || pathPart === '/') continue;
|
||
if (seenPaths.has(pathPart)) continue;
|
||
seenPaths.add(pathPart);
|
||
const isProduct = /\.html$/.test(pathPart);
|
||
rows.push({
|
||
path: pathPart,
|
||
url: cfg.base + pathPart,
|
||
slug_tokens: tokensFor(pathPart),
|
||
is_product: isProduct,
|
||
level: pathPart.split('/').filter(Boolean).length,
|
||
});
|
||
}
|
||
await client.query('BEGIN');
|
||
await client.query('DELETE FROM bridge_site_map_urls WHERE site = $1', [name]);
|
||
for (let i = 0; i < rows.length; i += 1000) {
|
||
const batch = rows.slice(i, i + 1000);
|
||
const values = [];
|
||
const params = [];
|
||
batch.forEach((r, idx) => {
|
||
const b = idx * 6;
|
||
values.push(`($${b + 1},$${b + 2},$${b + 3},$${b + 4},$${b + 5},$${b + 6})`);
|
||
params.push(name, r.path, r.url, r.slug_tokens, r.is_product, r.level);
|
||
});
|
||
await client.query(
|
||
`INSERT INTO bridge_site_map_urls (site, path, url, slug_tokens, is_product, level) VALUES ${values.join(',')}`,
|
||
params
|
||
);
|
||
}
|
||
await client.query('COMMIT');
|
||
const productCount = rows.filter((r) => r.is_product).length;
|
||
console.log(`[site-map] ${name}: total=${rows.length}, products=${productCount}, nav=${rows.length - productCount}`);
|
||
}
|
||
|
||
async function main() {
|
||
const client = new Client({ connectionString: PG_URL });
|
||
await client.connect();
|
||
await client.query('CREATE EXTENSION IF NOT EXISTS pg_trgm');
|
||
await client.query(`
|
||
CREATE TABLE IF NOT EXISTS bridge_site_map_urls (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
site TEXT NOT NULL,
|
||
path TEXT NOT NULL,
|
||
url TEXT NOT NULL,
|
||
slug_tokens TEXT NOT NULL,
|
||
is_product BOOLEAN NOT NULL DEFAULT FALSE,
|
||
level INTEGER NOT NULL DEFAULT 0,
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||
UNIQUE (site, path)
|
||
)
|
||
`);
|
||
await client.query('CREATE INDEX IF NOT EXISTS bridge_site_map_urls_site_idx ON bridge_site_map_urls (site)');
|
||
await client.query('CREATE INDEX IF NOT EXISTS bridge_site_map_urls_tokens_trgm_idx ON bridge_site_map_urls USING gin (slug_tokens gin_trgm_ops)');
|
||
const targets = process.argv.slice(2).filter((s) => SITES[s]);
|
||
for (const site of targets.length ? targets : ['www', 'us']) {
|
||
await buildSite(client, site);
|
||
}
|
||
await client.end();
|
||
}
|
||
|
||
main().catch((err) => {
|
||
console.error('[build-site-map]', err.message);
|
||
process.exit(1);
|
||
});
|