90 lines
3.0 KiB
JavaScript
90 lines
3.0 KiB
JavaScript
'use strict';
|
|
|
|
// KB Markdown 分块：与旧 bridge rag/chunker.js 保持一致的 parent/child 结构。
|
|
const MAX_CHILD_CHARS = 500;
|
|
|
|
function parseFrontmatter(text) {
|
|
const m = String(text).match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
|
|
if (!m) return { meta: {}, body: String(text) };
|
|
const meta = {};
|
|
const fm = m[1].split('\n');
|
|
for (let i = 0; i < fm.length; i += 1) {
|
|
const kv = fm[i].match(/^(\w+):\s*(.*)$/);
|
|
if (!kv) continue;
|
|
const key = kv[1];
|
|
let value = kv[2].trim();
|
|
if (value.startsWith('[')) {
|
|
value = value.replace(/[\[\]]/g, '').split(',').map((s) => s.trim().replace(/^["']|["']$/g, '')).filter(Boolean);
|
|
} else if (value === '') {
|
|
const arr = [];
|
|
while (i + 1 < fm.length && /^\s+-\s+/.test(fm[i + 1])) {
|
|
i += 1;
|
|
const item = fm[i].replace(/^\s+-\s+/, '').trim().replace(/^["']|["']$/g, '');
|
|
if (item) arr.push(item);
|
|
}
|
|
value = arr;
|
|
} else {
|
|
value = value.replace(/^["']|["']$/g, '');
|
|
}
|
|
meta[key] = value;
|
|
}
|
|
return { meta, body: m[2] };
|
|
}
|
|
|
|
function splitChunks(body) {
|
|
const chunks = [];
|
|
let current = null;
|
|
let section = '';
|
|
for (const line of String(body).split('\n')) {
|
|
const h2 = line.match(/^##\s+(.*)$/);
|
|
if (h2) { section = h2[1].trim(); continue; }
|
|
const q = line.match(/^###\s+Q:\s*(.*)$/);
|
|
if (q) {
|
|
if (current) chunks.push(current);
|
|
current = { question: q[1].trim(), answer: '', section };
|
|
} else if (current) {
|
|
current.answer += line + '\n';
|
|
}
|
|
}
|
|
if (current) chunks.push(current);
|
|
return chunks.map((c) => ({ ...c, answer: c.answer.trim() }));
|
|
}
|
|
|
|
function sourceUrls(meta) {
|
|
const raw = meta.source_urls || meta.source_url || [];
|
|
return (Array.isArray(raw) ? raw : [raw]).map(String).filter((u) => /^https?:\/\//.test(u));
|
|
}
|
|
|
|
function createParentChildChunks(chunk, meta, sourceFile) {
|
|
const keywords = Array.isArray(meta.keywords) ? meta.keywords : [];
|
|
const embText = [keywords.join(' '), chunk.question, chunk.answer].filter(Boolean).join('\n');
|
|
const parent = {
|
|
source_file: sourceFile,
|
|
title: meta.title || '',
|
|
category: meta.category || '',
|
|
section: chunk.section,
|
|
question: chunk.question,
|
|
answer: chunk.answer,
|
|
keywords,
|
|
source_urls: sourceUrls(meta),
|
|
chunk_type: 'parent',
|
|
embText,
|
|
};
|
|
if (chunk.answer.length <= MAX_CHILD_CHARS) return { parent, children: [] };
|
|
const children = (chunk.answer.match(new RegExp(`[^\\n]{1,${MAX_CHILD_CHARS}}`, 'g')) || [chunk.answer]).map((part) => ({
|
|
source_file: sourceFile,
|
|
title: meta.title || '',
|
|
category: meta.category || '',
|
|
section: chunk.section,
|
|
question: chunk.question,
|
|
answer: part,
|
|
keywords,
|
|
source_urls: sourceUrls(meta),
|
|
chunk_type: 'child',
|
|
embText: [keywords.join(' '), chunk.question, part].filter(Boolean).join('\n'),
|
|
}));
|
|
return { parent, children };
|
|
}
|
|
|
|
module.exports = { parseFrontmatter, splitChunks, createParentChildChunks, sourceUrls, MAX_CHILD_CHARS };
|