chore: CRM-NEW baseline

2026-10-23 10:35:28 +08:00
commit aa499f6dbf
111 changed files with 15126 additions and 0 deletions
@@ -0,0 +1,116 @@
+#!/usr/bin/env node
+'use strict';
+
+// 抓取官网 sitemap.xml（www/us），把全量 URL 落到 bridge 库 bridge_site_map_urls。
+// 数据只在服务端匹配，不进 LLM/向量库；商品搜索无结果时返回真实商品页链接。
+const fs = require('fs');
+const path = require('path');
+const { Client } = require('../bridge/node_modules/pg');
+
+const ROOT = path.resolve(__dirname, '..');
+const env = {};
+for (const line of fs.readFileSync(path.join(ROOT, '.env'), 'utf8').split('\n')) {
+  if (!line.includes('=') || line.trim().startsWith('#')) continue;
+  const i = line.indexOf('=');
+  env[line.slice(0, i).trim()] = line.slice(i + 1).trim();
+}
+const password = encodeURIComponent(env.POSTGRES_PASSWORD || '');
+const PG_URL = process.env.SITEMAP_PG_URL || `postgres://crmnew:${password}@127.0.0.1:5440/bridge`;
+
+const SITES = {
+  www: { base: 'https://www.yehwang.com', sitemap: 'https://www.yehwang.com/sitemap.xml' },
+  us: { base: 'https://us.yehwang.com', sitemap: 'https://us.yehwang.com/sitemap.xml' },
+};
+
+function tokensFor(pathPart) {
+  return pathPart
+    .replace(/\.html$/, '')
+    .split('/')
+    .filter(Boolean)
+    .map((seg) => seg.replace(/-?\d+$/, '').split('-').filter((w) => w && w.length > 1).join(' '))
+    .filter(Boolean)
+    .join(' ')
+    .toLowerCase();
+}
+
+async function fetchSitemap(url) {
+  const res = await fetch(url, {
+    headers: { 'User-Agent': 'Mozilla/5.0 (compatible; YehwangSiteMapBot/1.0)' },
+    signal: AbortSignal.timeout(90000),
+  });
+  if (!res.ok) throw new Error(`sitemap HTTP ${res.status}: ${url}`);
+  const xml = await res.text();
+  return [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1].trim()).filter(Boolean);
+}
+
+async function buildSite(client, name) {
+  const cfg = SITES[name];
+  const urls = await fetchSitemap(cfg.sitemap);
+  const rows = [];
+  const seenPaths = new Set();
+  for (const url of urls) {
+    if (!url.startsWith(cfg.base)) continue;
+    const pathPart = url.slice(cfg.base.length).split('?')[0];
+    if (!pathPart || pathPart === '/') continue;
+    if (seenPaths.has(pathPart)) continue;
+    seenPaths.add(pathPart);
+    const isProduct = /\.html$/.test(pathPart);
+    rows.push({
+      path: pathPart,
+      url: cfg.base + pathPart,
+      slug_tokens: tokensFor(pathPart),
+      is_product: isProduct,
+      level: pathPart.split('/').filter(Boolean).length,
+    });
+  }
+  await client.query('BEGIN');
+  await client.query('DELETE FROM bridge_site_map_urls WHERE site = $1', [name]);
+  for (let i = 0; i < rows.length; i += 1000) {
+    const batch = rows.slice(i, i + 1000);
+    const values = [];
+    const params = [];
+    batch.forEach((r, idx) => {
+      const b = idx * 6;
+      values.push(`($${b + 1},$${b + 2},$${b + 3},$${b + 4},$${b + 5},$${b + 6})`);
+      params.push(name, r.path, r.url, r.slug_tokens, r.is_product, r.level);
+    });
+    await client.query(
+      `INSERT INTO bridge_site_map_urls (site, path, url, slug_tokens, is_product, level) VALUES ${values.join(',')}`,
+      params
+    );
+  }
+  await client.query('COMMIT');
+  const productCount = rows.filter((r) => r.is_product).length;
+  console.log(`[site-map] ${name}: total=${rows.length}, products=${productCount}, nav=${rows.length - productCount}`);
+}
+
+async function main() {
+  const client = new Client({ connectionString: PG_URL });
+  await client.connect();
+  await client.query('CREATE EXTENSION IF NOT EXISTS pg_trgm');
+  await client.query(`
+    CREATE TABLE IF NOT EXISTS bridge_site_map_urls (
+      id BIGSERIAL PRIMARY KEY,
+      site TEXT NOT NULL,
+      path TEXT NOT NULL,
+      url TEXT NOT NULL,
+      slug_tokens TEXT NOT NULL,
+      is_product BOOLEAN NOT NULL DEFAULT FALSE,
+      level INTEGER NOT NULL DEFAULT 0,
+      created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+      UNIQUE (site, path)
+    )
+  `);
+  await client.query('CREATE INDEX IF NOT EXISTS bridge_site_map_urls_site_idx ON bridge_site_map_urls (site)');
+  await client.query('CREATE INDEX IF NOT EXISTS bridge_site_map_urls_tokens_trgm_idx ON bridge_site_map_urls USING gin (slug_tokens gin_trgm_ops)');
+  const targets = process.argv.slice(2).filter((s) => SITES[s]);
+  for (const site of targets.length ? targets : ['www', 'us']) {
+    await buildSite(client, site);
+  }
+  await client.end();
+}
+
+main().catch((err) => {
+  console.error('[build-site-map]', err.message);
+  process.exit(1);
+});