feat: add online FAQ/KB sync without old bridge

2026-10-23 11:29:43 +08:00
parent f526696f93
commit 81b49c9350
24 changed files with 1793 additions and 5 deletions
@@ -0,0 +1,89 @@
+'use strict';
+
+// KB Markdown 分块：与旧 bridge rag/chunker.js 保持一致的 parent/child 结构。
+const MAX_CHILD_CHARS = 500;
+
+function parseFrontmatter(text) {
+  const m = String(text).match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
+  if (!m) return { meta: {}, body: String(text) };
+  const meta = {};
+  const fm = m[1].split('\n');
+  for (let i = 0; i < fm.length; i += 1) {
+    const kv = fm[i].match(/^(\w+):\s*(.*)$/);
+    if (!kv) continue;
+    const key = kv[1];
+    let value = kv[2].trim();
+    if (value.startsWith('[')) {
+      value = value.replace(/[\[\]]/g, '').split(',').map((s) => s.trim().replace(/^["']|["']$/g, '')).filter(Boolean);
+    } else if (value === '') {
+      const arr = [];
+      while (i + 1 < fm.length && /^\s+-\s+/.test(fm[i + 1])) {
+        i += 1;
+        const item = fm[i].replace(/^\s+-\s+/, '').trim().replace(/^["']|["']$/g, '');
+        if (item) arr.push(item);
+      }
+      value = arr;
+    } else {
+      value = value.replace(/^["']|["']$/g, '');
+    }
+    meta[key] = value;
+  }
+  return { meta, body: m[2] };
+}
+
+function splitChunks(body) {
+  const chunks = [];
+  let current = null;
+  let section = '';
+  for (const line of String(body).split('\n')) {
+    const h2 = line.match(/^##\s+(.*)$/);
+    if (h2) { section = h2[1].trim(); continue; }
+    const q = line.match(/^###\s+Q:\s*(.*)$/);
+    if (q) {
+      if (current) chunks.push(current);
+      current = { question: q[1].trim(), answer: '', section };
+    } else if (current) {
+      current.answer += line + '\n';
+    }
+  }
+  if (current) chunks.push(current);
+  return chunks.map((c) => ({ ...c, answer: c.answer.trim() }));
+}
+
+function sourceUrls(meta) {
+  const raw = meta.source_urls || meta.source_url || [];
+  return (Array.isArray(raw) ? raw : [raw]).map(String).filter((u) => /^https?:\/\//.test(u));
+}
+
+function createParentChildChunks(chunk, meta, sourceFile) {
+  const keywords = Array.isArray(meta.keywords) ? meta.keywords : [];
+  const embText = [keywords.join(' '), chunk.question, chunk.answer].filter(Boolean).join('\n');
+  const parent = {
+    source_file: sourceFile,
+    title: meta.title || '',
+    category: meta.category || '',
+    section: chunk.section,
+    question: chunk.question,
+    answer: chunk.answer,
+    keywords,
+    source_urls: sourceUrls(meta),
+    chunk_type: 'parent',
+    embText,
+  };
+  if (chunk.answer.length <= MAX_CHILD_CHARS) return { parent, children: [] };
+  const children = (chunk.answer.match(new RegExp(`[^\\n]{1,${MAX_CHILD_CHARS}}`, 'g')) || [chunk.answer]).map((part) => ({
+    source_file: sourceFile,
+    title: meta.title || '',
+    category: meta.category || '',
+    section: chunk.section,
+    question: chunk.question,
+    answer: part,
+    keywords,
+    source_urls: sourceUrls(meta),
+    chunk_type: 'child',
+    embText: [keywords.join(' '), chunk.question, part].filter(Boolean).join('\n'),
+  }));
+  return { parent, children };
+}
+
+module.exports = { parseFrontmatter, splitChunks, createParentChildChunks, sourceUrls, MAX_CHILD_CHARS };