crm-ai/scripts/build-site-map.js

#!/usr/bin/env node
'use strict';

// 抓取官网 sitemap.xml（www/us），把全量 URL 落到 bridge 库 bridge_site_map_urls。
// 数据只在服务端匹配，不进 LLM/向量库；商品搜索无结果时返回真实商品页链接。
const fs = require('fs');
const path = require('path');
const { Client } = require('../bridge/node_modules/pg');

const ROOT = path.resolve(__dirname, '..');
const env = {};
for (const line of fs.readFileSync(path.join(ROOT, '.env'), 'utf8').split('\n')) {
  if (!line.includes('=') || line.trim().startsWith('#')) continue;
  const i = line.indexOf('=');
  env[line.slice(0, i).trim()] = line.slice(i + 1).trim();
}
const password = encodeURIComponent(env.POSTGRES_PASSWORD || '');
const PG_URL = process.env.SITEMAP_PG_URL || `postgres://crmnew:${password}@127.0.0.1:5440/bridge`;

const SITES = {
  www: { base: 'https://www.yehwang.com', sitemap: 'https://www.yehwang.com/sitemap.xml' },
  us: { base: 'https://us.yehwang.com', sitemap: 'https://us.yehwang.com/sitemap.xml' },
};

function tokensFor(pathPart) {
  return pathPart
    .replace(/\.html$/, '')
    .split('/')
    .filter(Boolean)
    .map((seg) => seg.replace(/-?\d+$/, '').split('-').filter((w) => w && w.length > 1).join(' '))
    .filter(Boolean)
    .join(' ')
    .toLowerCase();
}

function parseLocs(xml) {
  return [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1].trim()).filter(Boolean);
}

// 官网 sitemap 受阿里云 WAF JS 挑战保护，直接 fetch 会返回挑战 HTML（0 个 <loc>）。
// 支持从本地缓存的 XML 读取：SITEMAP_XML_<SITE> 环境变量，或 var/sitemap-<site>.xml。
function localSitemapFile(name) {
  const fromEnv = process.env['SITEMAP_XML_' + name.toUpperCase()];
  if (fromEnv && fs.existsSync(fromEnv)) return fromEnv;
  const p = path.join(ROOT, 'var', `sitemap-${name}.xml`);
  return fs.existsSync(p) ? p : null;
}

async function fetchSitemap(name, url) {
  const local = localSitemapFile(name);
  if (local) {
    console.log(`[site-map] ${name}: 使用本地 sitemap ${local}`);
    return parseLocs(await fs.promises.readFile(local, 'utf8'));
  }
  const res = await fetch(url, {
    headers: { 'User-Agent': 'Mozilla/5.0 (compatible; YehwangSiteMapBot/1.0)' },
    signal: AbortSignal.timeout(90000),
  });
  if (!res.ok) throw new Error(`sitemap HTTP ${res.status}: ${url}`);
  const xml = await res.text();
  const locs = parseLocs(xml);
  if (!locs.length) {
    throw new Error(`sitemap 返回 0 个 <loc>（可能被 WAF 挑战）：先下载 XML 到 var/sitemap-${name}.xml，或设置 SITEMAP_XML_${name.toUpperCase()}`);
  }
  return locs;
}

async function buildSite(client, name) {
  const cfg = SITES[name];
  const urls = await fetchSitemap(name, cfg.sitemap);
  const rows = [];
  const seenPaths = new Set();
  for (const url of urls) {
    if (!url.startsWith(cfg.base)) continue;
    const pathPart = url.slice(cfg.base.length).split('?')[0];
    if (!pathPart || pathPart === '/') continue;
    if (seenPaths.has(pathPart)) continue;
    seenPaths.add(pathPart);
    const isProduct = /\.html$/.test(pathPart);
    rows.push({
      path: pathPart,
      url: cfg.base + pathPart,
      slug_tokens: tokensFor(pathPart),
      is_product: isProduct,
      level: pathPart.split('/').filter(Boolean).length,
    });
  }
  await client.query('BEGIN');
  await client.query('DELETE FROM bridge_site_map_urls WHERE site = $1', [name]);
  for (let i = 0; i < rows.length; i += 1000) {
    const batch = rows.slice(i, i + 1000);
    const values = [];
    const params = [];
    batch.forEach((r, idx) => {
      const b = idx * 6;
      values.push(`($${b + 1},$${b + 2},$${b + 3},$${b + 4},$${b + 5},$${b + 6})`);
      params.push(name, r.path, r.url, r.slug_tokens, r.is_product, r.level);
    });
    await client.query(
      `INSERT INTO bridge_site_map_urls (site, path, url, slug_tokens, is_product, level) VALUES ${values.join(',')}`,
      params
    );
  }
  await client.query('COMMIT');
  const productCount = rows.filter((r) => r.is_product).length;
  console.log(`[site-map] ${name}: total=${rows.length}, products=${productCount}, nav=${rows.length - productCount}`);
}

async function main() {
  const client = new Client({ connectionString: PG_URL });
  await client.connect();
  await client.query('CREATE EXTENSION IF NOT EXISTS pg_trgm');
  await client.query(`
    CREATE TABLE IF NOT EXISTS bridge_site_map_urls (
      id BIGSERIAL PRIMARY KEY,
      site TEXT NOT NULL,
      path TEXT NOT NULL,
      url TEXT NOT NULL,
      slug_tokens TEXT NOT NULL,
      is_product BOOLEAN NOT NULL DEFAULT FALSE,
      level INTEGER NOT NULL DEFAULT 0,
      created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
      UNIQUE (site, path)
    )
  `);
  await client.query('CREATE INDEX IF NOT EXISTS bridge_site_map_urls_site_idx ON bridge_site_map_urls (site)');
  await client.query('CREATE INDEX IF NOT EXISTS bridge_site_map_urls_tokens_trgm_idx ON bridge_site_map_urls USING gin (slug_tokens gin_trgm_ops)');
  const targets = process.argv.slice(2).filter((s) => SITES[s]);
  for (const site of targets.length ? targets : ['www', 'us']) {
    await buildSite(client, site);
  }
  await client.end();
}

main().catch((err) => {
  console.error('[build-site-map]', err.message);
  process.exit(1);
});