feat: add online FAQ/KB sync without old bridge
This commit is contained in:
wangliang
@@ -0,0 +1,81 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
function stripHtml(value) {
|
||||
return String(value || '')
|
||||
.replace(/<br\s*\/?>/gi, '\n')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function extractFaqFromLdJson(html, max = 50) {
|
||||
const re = /<script[^>]*type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
|
||||
let m;
|
||||
while ((m = re.exec(String(html)))) {
|
||||
try {
|
||||
const data = JSON.parse(m[1].trim());
|
||||
const candidates = Array.isArray(data) ? data : [data];
|
||||
for (const item of candidates) {
|
||||
const main = item && (item.mainEntity || (item['@type'] === 'FAQPage' && item.mainEntity));
|
||||
if (!Array.isArray(main)) continue;
|
||||
const out = main
|
||||
.filter((q) => q && q.name && q.acceptedAnswer && q.acceptedAnswer.text)
|
||||
.map((q) => ({ q: stripHtml(q.name), a: stripHtml(q.acceptedAnswer.text) }))
|
||||
.filter((q) => q.q && q.a)
|
||||
.slice(0, max);
|
||||
if (out.length) return out;
|
||||
}
|
||||
} catch {
|
||||
// 非 JSON-LD 块忽略
|
||||
}
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
function loadFaqConfig(file) {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
} catch {
|
||||
return { enabled: false, pages: [] };
|
||||
}
|
||||
}
|
||||
|
||||
function createFaqFetcher(config = {}) {
|
||||
const pages = config.pages || [];
|
||||
const timeoutMs = config.timeoutMs || 15000;
|
||||
const max = config.maxFaqsPerPage || 50;
|
||||
|
||||
async function fetchPage(url) {
|
||||
const res = await fetch(url, {
|
||||
headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36', Accept: 'text/html,application/xhtml+xml' },
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
});
|
||||
if (!res.ok) throw new Error(`fetch ${url} failed: HTTP ${res.status}`);
|
||||
return extractFaqFromLdJson(await res.text(), max);
|
||||
}
|
||||
|
||||
async function fetchAll() {
|
||||
const out = [];
|
||||
for (const page of pages) {
|
||||
try {
|
||||
const questions = await fetchPage(page.url);
|
||||
out.push({ ...page, questions });
|
||||
} catch (err) {
|
||||
out.push({ ...page, questions: [], error: err.message });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
return { fetchPage, fetchAll, pages };
|
||||
}
|
||||
|
||||
module.exports = { createFaqFetcher, loadFaqConfig, extractFaqFromLdJson, stripHtml };
|
||||
@@ -0,0 +1,72 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs');
|
||||
|
||||
function escapeRe(value) {
|
||||
return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
|
||||
function normalizeQuestion(value) {
|
||||
return String(value || '').toLowerCase().replace(/[^\p{L}\p{N}]+/gu, ' ').trim();
|
||||
}
|
||||
|
||||
// 用官网 FAQ 更新本地 markdown：保留 frontmatter、章节标题和 Q 顺序，
|
||||
// 只替换已有问题的答案，新问题追加到文件末尾。
|
||||
function updateMarkdownFile(filePath, questions) {
|
||||
if (!fs.existsSync(filePath)) return { changed: false, reason: 'file_missing', added: 0, updated: 0 };
|
||||
let content = fs.readFileSync(filePath, 'utf8');
|
||||
const bodyStart = (content.match(/^---\n[\s\S]*?\n---\n/) || [''])[0].length;
|
||||
const body = content.slice(bodyStart);
|
||||
|
||||
const existing = new Map();
|
||||
let currentQ = null;
|
||||
let answer = [];
|
||||
const flush = () => {
|
||||
if (currentQ) existing.set(normalizeQuestion(currentQ), { question: currentQ, answer: answer.join('\n').trim() });
|
||||
};
|
||||
for (const line of body.split('\n')) {
|
||||
const q = line.match(/^###\s+Q:\s*(.*)$/);
|
||||
if (q) {
|
||||
flush();
|
||||
currentQ = q[1].trim();
|
||||
answer = [];
|
||||
} else if (currentQ) {
|
||||
answer.push(line);
|
||||
}
|
||||
}
|
||||
flush();
|
||||
|
||||
let changed = false;
|
||||
let added = 0;
|
||||
let updated = 0;
|
||||
const additions = [];
|
||||
for (const item of questions) {
|
||||
if (!item.q || !item.a) continue;
|
||||
const key = normalizeQuestion(item.q);
|
||||
if (existing.has(key)) {
|
||||
const prev = existing.get(key);
|
||||
if (prev.answer !== item.a) {
|
||||
const re = new RegExp(`(### Q: ${escapeRe(prev.question)}\\n)([\\s\\S]*?)(?=\\n### Q:|$)`, 'm');
|
||||
if (re.test(content)) {
|
||||
content = content.replace(re, (_m, prefix) => `${prefix}${item.a}\n`);
|
||||
prev.answer = item.a;
|
||||
changed = true;
|
||||
updated += 1;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
existing.set(key, { question: item.q, answer: item.a });
|
||||
additions.push(`### Q: ${item.q}\n${item.a}\n`);
|
||||
added += 1;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (additions.length) {
|
||||
content = content.replace(/\s+$/, '') + '\n\n' + additions.join('\n');
|
||||
}
|
||||
if (changed) fs.writeFileSync(filePath, content, 'utf8');
|
||||
return { changed, added, updated };
|
||||
}
|
||||
|
||||
module.exports = { updateMarkdownFile };
|
||||
@@ -0,0 +1,89 @@
|
||||
'use strict';
|
||||
|
||||
// KB Markdown 分块：与旧 bridge rag/chunker.js 保持一致的 parent/child 结构。
|
||||
const MAX_CHILD_CHARS = 500;
|
||||
|
||||
function parseFrontmatter(text) {
|
||||
const m = String(text).match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
|
||||
if (!m) return { meta: {}, body: String(text) };
|
||||
const meta = {};
|
||||
const fm = m[1].split('\n');
|
||||
for (let i = 0; i < fm.length; i += 1) {
|
||||
const kv = fm[i].match(/^(\w+):\s*(.*)$/);
|
||||
if (!kv) continue;
|
||||
const key = kv[1];
|
||||
let value = kv[2].trim();
|
||||
if (value.startsWith('[')) {
|
||||
value = value.replace(/[\[\]]/g, '').split(',').map((s) => s.trim().replace(/^["']|["']$/g, '')).filter(Boolean);
|
||||
} else if (value === '') {
|
||||
const arr = [];
|
||||
while (i + 1 < fm.length && /^\s+-\s+/.test(fm[i + 1])) {
|
||||
i += 1;
|
||||
const item = fm[i].replace(/^\s+-\s+/, '').trim().replace(/^["']|["']$/g, '');
|
||||
if (item) arr.push(item);
|
||||
}
|
||||
value = arr;
|
||||
} else {
|
||||
value = value.replace(/^["']|["']$/g, '');
|
||||
}
|
||||
meta[key] = value;
|
||||
}
|
||||
return { meta, body: m[2] };
|
||||
}
|
||||
|
||||
function splitChunks(body) {
|
||||
const chunks = [];
|
||||
let current = null;
|
||||
let section = '';
|
||||
for (const line of String(body).split('\n')) {
|
||||
const h2 = line.match(/^##\s+(.*)$/);
|
||||
if (h2) { section = h2[1].trim(); continue; }
|
||||
const q = line.match(/^###\s+Q:\s*(.*)$/);
|
||||
if (q) {
|
||||
if (current) chunks.push(current);
|
||||
current = { question: q[1].trim(), answer: '', section };
|
||||
} else if (current) {
|
||||
current.answer += line + '\n';
|
||||
}
|
||||
}
|
||||
if (current) chunks.push(current);
|
||||
return chunks.map((c) => ({ ...c, answer: c.answer.trim() }));
|
||||
}
|
||||
|
||||
function sourceUrls(meta) {
|
||||
const raw = meta.source_urls || meta.source_url || [];
|
||||
return (Array.isArray(raw) ? raw : [raw]).map(String).filter((u) => /^https?:\/\//.test(u));
|
||||
}
|
||||
|
||||
function createParentChildChunks(chunk, meta, sourceFile) {
|
||||
const keywords = Array.isArray(meta.keywords) ? meta.keywords : [];
|
||||
const embText = [keywords.join(' '), chunk.question, chunk.answer].filter(Boolean).join('\n');
|
||||
const parent = {
|
||||
source_file: sourceFile,
|
||||
title: meta.title || '',
|
||||
category: meta.category || '',
|
||||
section: chunk.section,
|
||||
question: chunk.question,
|
||||
answer: chunk.answer,
|
||||
keywords,
|
||||
source_urls: sourceUrls(meta),
|
||||
chunk_type: 'parent',
|
||||
embText,
|
||||
};
|
||||
if (chunk.answer.length <= MAX_CHILD_CHARS) return { parent, children: [] };
|
||||
const children = (chunk.answer.match(new RegExp(`[^\\n]{1,${MAX_CHILD_CHARS}}`, 'g')) || [chunk.answer]).map((part) => ({
|
||||
source_file: sourceFile,
|
||||
title: meta.title || '',
|
||||
category: meta.category || '',
|
||||
section: chunk.section,
|
||||
question: chunk.question,
|
||||
answer: part,
|
||||
keywords,
|
||||
source_urls: sourceUrls(meta),
|
||||
chunk_type: 'child',
|
||||
embText: [keywords.join(' '), chunk.question, part].filter(Boolean).join('\n'),
|
||||
}));
|
||||
return { parent, children };
|
||||
}
|
||||
|
||||
module.exports = { parseFrontmatter, splitChunks, createParentChildChunks, sourceUrls, MAX_CHILD_CHARS };
|
||||
@@ -0,0 +1,180 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const { parseFrontmatter, splitChunks, createParentChildChunks } = require('./kb-chunker');
|
||||
|
||||
// 新 bridge 独立 KB 索引器：读本地 kb/*.md，生成 parent/child + embedding，
|
||||
// staging 表校验后原子切换 kb_docs，旧表保留为 kb_docs_v<timestamp>。
|
||||
function createKbIndexer({ pool, embeddingClient, kbDir, dim = 1024, logger = console }) {
|
||||
const DIM = Number(dim) || 1024;
|
||||
const KB_DIR = kbDir || path.resolve(__dirname, '..', 'kb');
|
||||
|
||||
async function ensureBase() {
|
||||
await pool.query('CREATE EXTENSION IF NOT EXISTS vector');
|
||||
await pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS kb_sync_runs (
|
||||
id SERIAL PRIMARY KEY,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
source_hash TEXT,
|
||||
parent_count INT,
|
||||
child_count INT,
|
||||
backup_table TEXT,
|
||||
note TEXT
|
||||
)
|
||||
`);
|
||||
}
|
||||
|
||||
function listFiles() {
|
||||
return fs.readdirSync(KB_DIR)
|
||||
.filter((f) => f.endsWith('.md') && /^\d+/.test(f))
|
||||
.sort();
|
||||
}
|
||||
|
||||
function sourceHash(files) {
|
||||
const h = crypto.createHash('sha256');
|
||||
for (const f of files) {
|
||||
h.update(f);
|
||||
h.update(fs.readFileSync(path.join(KB_DIR, f)));
|
||||
}
|
||||
return h.digest('hex');
|
||||
}
|
||||
|
||||
async function createStaging() {
|
||||
await pool.query('DROP TABLE IF EXISTS kb_docs_staging CASCADE');
|
||||
await pool.query(`
|
||||
CREATE TABLE kb_docs_staging (
|
||||
id SERIAL PRIMARY KEY,
|
||||
source_file TEXT,
|
||||
title TEXT,
|
||||
category TEXT,
|
||||
section TEXT,
|
||||
question TEXT,
|
||||
answer TEXT,
|
||||
keywords TEXT[],
|
||||
source_urls TEXT[],
|
||||
text_vec vector(${DIM}),
|
||||
parent_id INTEGER,
|
||||
chunk_type TEXT DEFAULT 'parent',
|
||||
keywords_text TEXT,
|
||||
text_tsv tsvector
|
||||
)
|
||||
`);
|
||||
}
|
||||
|
||||
async function insertDoc(table, doc, parentId) {
|
||||
const text = [doc.title, doc.category, doc.section, doc.question, doc.answer, (doc.keywords || []).join(' ')].filter(Boolean).join(' ');
|
||||
const r = await pool.query(
|
||||
`INSERT INTO ${table}
|
||||
(source_file,title,category,section,question,answer,keywords,source_urls,text_vec,parent_id,chunk_type,keywords_text,text_tsv)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9::vector,$10,$11,$12,to_tsvector('english',$13))
|
||||
RETURNING id`,
|
||||
[doc.source_file, doc.title, doc.category, doc.section, doc.question, doc.answer,
|
||||
doc.keywords || [], doc.source_urls || [], embeddingClient.vecStr(doc.vec), parentId || null,
|
||||
doc.chunk_type, (doc.keywords || []).join(' '), text]
|
||||
);
|
||||
return r.rows[0].id;
|
||||
}
|
||||
|
||||
async function indexFile(table, file) {
|
||||
const raw = fs.readFileSync(path.join(KB_DIR, file), 'utf8');
|
||||
const { meta, body } = parseFrontmatter(raw);
|
||||
const chunks = splitChunks(body);
|
||||
let parents = 0;
|
||||
let children = 0;
|
||||
for (const chunk of chunks) {
|
||||
const { parent, children: childChunks } = createParentChildChunks(chunk, meta, file);
|
||||
parent.vec = await embeddingClient.embedQuery(parent.embText);
|
||||
const parentId = await insertDoc(table, parent, null);
|
||||
parents += 1;
|
||||
for (const child of childChunks) {
|
||||
child.vec = await embeddingClient.embedQuery(child.embText);
|
||||
await insertDoc(table, child, parentId);
|
||||
children += 1;
|
||||
}
|
||||
}
|
||||
return { parents, children };
|
||||
}
|
||||
|
||||
async function createStagingIndexes() {
|
||||
await pool.query('CREATE INDEX kb_docs_staging_vec_idx ON kb_docs_staging USING hnsw (text_vec vector_cosine_ops)');
|
||||
await pool.query('CREATE INDEX kb_docs_staging_tsv_idx ON kb_docs_staging USING GIN (text_tsv)');
|
||||
await pool.query('CREATE INDEX kb_docs_staging_parent_idx ON kb_docs_staging (parent_id) WHERE parent_id IS NOT NULL');
|
||||
}
|
||||
|
||||
async function atomicSwitch() {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
const has = await client.query("SELECT to_regclass('public.kb_docs') IS NOT NULL AS exists");
|
||||
let backup = null;
|
||||
if (has.rows[0].exists) {
|
||||
const stamp = new Date().toISOString().replace(/[-:TZ.]/g, '').slice(0, 14);
|
||||
backup = `kb_docs_v${stamp}`;
|
||||
let suffix = 0;
|
||||
while ((await client.query('SELECT to_regclass($1) IS NOT NULL AS exists', [`public.${backup}`])).rows[0].exists) {
|
||||
suffix += 1;
|
||||
backup = `kb_docs_v${stamp}_${suffix}`;
|
||||
}
|
||||
await client.query(`ALTER TABLE kb_docs RENAME TO ${backup}`);
|
||||
await client.query(`ALTER INDEX IF EXISTS kb_docs_vec_idx RENAME TO ${backup}_vec_idx`);
|
||||
await client.query(`ALTER INDEX IF EXISTS kb_docs_tsv_idx RENAME TO ${backup}_tsv_idx`);
|
||||
await client.query(`ALTER INDEX IF EXISTS kb_docs_parent_idx RENAME TO ${backup}_parent_idx`);
|
||||
}
|
||||
await client.query('ALTER TABLE kb_docs_staging RENAME TO kb_docs');
|
||||
await client.query('ALTER INDEX IF EXISTS kb_docs_staging_vec_idx RENAME TO kb_docs_vec_idx');
|
||||
await client.query('ALTER INDEX IF EXISTS kb_docs_staging_tsv_idx RENAME TO kb_docs_tsv_idx');
|
||||
await client.query('ALTER INDEX IF EXISTS kb_docs_staging_parent_idx RENAME TO kb_docs_parent_idx');
|
||||
await client.query('COMMIT');
|
||||
return backup;
|
||||
} catch (err) {
|
||||
await client.query('ROLLBACK');
|
||||
throw err;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function reindex({ note = 'kb sync' } = {}) {
|
||||
const lockClient = await pool.connect();
|
||||
const lock = await lockClient.query("SELECT pg_try_advisory_lock(hashtext('bridge_kb_sync')) AS locked");
|
||||
if (!lock.rows[0].locked) {
|
||||
lockClient.release();
|
||||
throw new Error('another KB sync is running');
|
||||
}
|
||||
try {
|
||||
await ensureBase();
|
||||
const files = listFiles();
|
||||
if (!files.length) throw new Error(`no KB markdown files in ${KB_DIR}`);
|
||||
const hash = sourceHash(files);
|
||||
await createStaging();
|
||||
let parents = 0;
|
||||
let children = 0;
|
||||
for (const file of files) {
|
||||
const res = await indexFile('kb_docs_staging', file);
|
||||
parents += res.parents;
|
||||
children += res.children;
|
||||
}
|
||||
if (parents === 0) throw new Error('indexed 0 parent chunks');
|
||||
await createStagingIndexes();
|
||||
const backup = await atomicSwitch();
|
||||
await pool.query(
|
||||
'INSERT INTO kb_sync_runs (source_hash,parent_count,child_count,backup_table,note) VALUES ($1,$2,$3,$4,$5)',
|
||||
[hash, parents, children, backup, note]
|
||||
);
|
||||
logger.log(JSON.stringify({ ts: new Date().toISOString(), kb_sync: 'done', parents, children, backup, files: files.length }));
|
||||
return { parents, children, backup, sourceHash: hash, files: files.length };
|
||||
} catch (err) {
|
||||
await pool.query('DROP TABLE IF EXISTS kb_docs_staging CASCADE').catch(() => {});
|
||||
throw err;
|
||||
} finally {
|
||||
await lockClient.query("SELECT pg_advisory_unlock(hashtext('bridge_kb_sync'))").catch(() => {});
|
||||
lockClient.release();
|
||||
}
|
||||
}
|
||||
|
||||
return { reindex, listFiles, sourceHash, KB_DIR };
|
||||
}
|
||||
|
||||
module.exports = { createKbIndexer };
|
||||
Reference in New Issue
Block a user