feat: port quality, escalation, KB version, site-map KB, experiments and metrics
This commit is contained in:
wangliang
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"experiments": []
|
||||
}
|
||||
+1003
-322
File diff suppressed because it is too large
Load Diff
+120
-3
@@ -12,10 +12,17 @@ const { markdownToHtml } = require('./markdown');
|
||||
const { guardReply } = require('./output-guard');
|
||||
const { quickClassify, detectEmotion, INTENTS } = require('./intent');
|
||||
const { createContextManager } = require('./context');
|
||||
const { createReranker } = require('./reranker');
|
||||
const { detectInjection, markToolResult, INJECTION_CAUTION_PROMPT } = require('./security');
|
||||
const { hasBuyingIntent, missingLeadContacts, detectLanguage, appendLeadFollowup, alreadyAsksForLead } = require('./lead-intent');
|
||||
const { createLlmCache } = require('./llm-cache');
|
||||
const { checkCitation } = require('./citation-checker');
|
||||
const { createFlywheelWriter } = require('./flywheel');
|
||||
const { languageMandate } = require('./language');
|
||||
const { createEscalation } = require('./escalation');
|
||||
const { createExperimentManager } = require('./experiments');
|
||||
const crypto = require('crypto');
|
||||
const prom = require('./prom');
|
||||
|
||||
const MAX_TOOL_STEPS = 4;
|
||||
|
||||
@@ -40,7 +47,18 @@ function createAgent(cfg, deps = {}) {
|
||||
const metrics = deps.metrics || createMetrics(cfg);
|
||||
const contextManager = deps.contextManager || createContextManager();
|
||||
const llmCache = deps.llmCache || createLlmCache(cfg);
|
||||
const tools = deps.tools || createTools(cfg, { verification, retriever, client });
|
||||
const escalation = deps.escalation || createEscalation({ client, handoffUserId: cfg.handoffUserId || 1 });
|
||||
const experiments = deps.experiments || createExperimentManager(cfg);
|
||||
const flywheel = deps.flywheel || createFlywheelWriter({
|
||||
pool: metrics.pool,
|
||||
callLLM: async (messages, system, maxTokens) => {
|
||||
const msgs = system ? [{ role: 'system', content: system }, ...messages] : messages;
|
||||
const res = await llm.chat({ messages: msgs, maxTokens: maxTokens || 300 });
|
||||
return { text: res.content || '' };
|
||||
},
|
||||
});
|
||||
const reranker = deps.reranker || createReranker(llm);
|
||||
const tools = deps.tools || createTools(cfg, { verification, retriever, client, reranker });
|
||||
const leadStore = deps.leadStore || (tools && tools.leadStore) || null;
|
||||
const byName = new Map(tools.map((t) => [t.name, t]));
|
||||
const toolDefs = tools.map((t) => ({
|
||||
@@ -87,6 +105,7 @@ function createAgent(cfg, deps = {}) {
|
||||
const startedAt = Date.now();
|
||||
const stats = { llmCalls: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0, toolCalls: [], toolResults: [] };
|
||||
const profile = resolveSiteProfile(cfg, { inboxId: String(conv.inbox_id || '') });
|
||||
const variant = experiments.getVariant(conv.uuid);
|
||||
const identity = await identityFor(conv);
|
||||
const ctx = {
|
||||
verified: identity.verified,
|
||||
@@ -97,6 +116,7 @@ function createAgent(cfg, deps = {}) {
|
||||
};
|
||||
|
||||
const currentText = (webhookMessage.text_content || stripHtml(webhookMessage.content)).trim();
|
||||
ctx.language = detectLanguage(currentText);
|
||||
const messages = await buildMessages(conv, webhookMessage);
|
||||
if (!messages.length) return null;
|
||||
const injection = detectInjection(currentText);
|
||||
@@ -107,6 +127,8 @@ function createAgent(cfg, deps = {}) {
|
||||
channel: conv.channel || 'livechat',
|
||||
}) + (injection.level === 'high' ? INJECTION_CAUTION_PROMPT : '');
|
||||
|
||||
systemContent += languageMandate(currentText);
|
||||
|
||||
// 购买意向：LLM 之前先给硬指令，LLM 之后还有程序化兜底
|
||||
let leadState = null;
|
||||
if (leadStore) {
|
||||
@@ -126,6 +148,8 @@ function createAgent(cfg, deps = {}) {
|
||||
// 预检索：政策问题先查 KB，商品问题先查商品，避免模型跳过工具直接编。
|
||||
const intent = quickClassify(currentText);
|
||||
const emotion = detectEmotion(currentText);
|
||||
const emotionStrategy = escalation.applyEmotionStrategy(emotion);
|
||||
const maxSteps = emotionStrategy.maxToolTurns || MAX_TOOL_STEPS;
|
||||
if (emotion !== 'neutral') {
|
||||
const toneLine = emotion === 'angry'
|
||||
? 'TONE: The customer is angry. Acknowledge their frustration, be empathetic, and escalate/hand off if needed.'
|
||||
@@ -180,12 +204,19 @@ function createAgent(cfg, deps = {}) {
|
||||
toolCalls: stats.toolCalls, audit: { ...auditAnswer({ finalContent: cached, toolResults: stats.toolResults, toolCalls: stats.toolCalls }), cache_hit: true },
|
||||
replyLength: cached.length,
|
||||
});
|
||||
prom.inc('bridge_ai_cache_hits_total');
|
||||
console.log(JSON.stringify({ ts: new Date().toISOString(), agent: 'cache_hit', conversation: conv.uuid, length: cached.length }));
|
||||
return cached;
|
||||
}
|
||||
}
|
||||
for (let step = 0; step < MAX_TOOL_STEPS; step += 1) {
|
||||
const response = await llm.chat({ messages, tools: toolDefs, temperature: 0.3, maxTokens: 1200 });
|
||||
for (let step = 0; step < maxSteps; step += 1) {
|
||||
prom.inc('bridge_ai_llm_calls_total');
|
||||
const response = await llm.chat({
|
||||
messages,
|
||||
tools: toolDefs,
|
||||
temperature: (variant.overriddenParams && variant.overriddenParams.temperature) ?? 0.3,
|
||||
maxTokens: (variant.overriddenParams && variant.overriddenParams.maxTokens) ?? 1200,
|
||||
});
|
||||
stats.llmCalls += 1;
|
||||
if (response.usage) {
|
||||
stats.promptTokens += Number(response.usage.prompt_tokens) || 0;
|
||||
@@ -210,6 +241,7 @@ function createAgent(cfg, deps = {}) {
|
||||
} catch (err) {
|
||||
result = { error: err.code || err.message || 'tool_failed', message: err.message || String(err) };
|
||||
}
|
||||
prom.inc('bridge_ai_tool_calls_total', { tool: name, ok: !(result && result.error) });
|
||||
stats.toolCalls.push({ name, ms: Date.now() - started, ok: !(result && result.error) });
|
||||
stats.toolResults.push(result);
|
||||
console.log(JSON.stringify({
|
||||
@@ -256,6 +288,7 @@ function createAgent(cfg, deps = {}) {
|
||||
toolCalls: stats.toolCalls,
|
||||
lang: detectLanguage(currentText),
|
||||
});
|
||||
if (guard.blocked) prom.inc('bridge_ai_guard_blocked_total', { reason: (guard.reasons || []).join('|') || 'unknown' });
|
||||
if (guard.blocked || guard.removed_urls.length) {
|
||||
console.log(JSON.stringify({
|
||||
ts: new Date().toISOString(),
|
||||
@@ -270,6 +303,57 @@ function createAgent(cfg, deps = {}) {
|
||||
}
|
||||
finalContent = guard.content;
|
||||
|
||||
// 引用一致性检查（语义/数字覆盖）+ 飞轮
|
||||
const kbHits = [];
|
||||
const kbSources = [];
|
||||
(stats.toolCalls || []).forEach((call, i) => {
|
||||
if (call.name !== 'search_kb') return;
|
||||
const result = stats.toolResults[i];
|
||||
for (const hit of (result && result.results) || []) {
|
||||
kbHits.push(hit);
|
||||
for (const u of hit.source_urls || []) kbSources.push(u);
|
||||
}
|
||||
});
|
||||
const citation = checkCitation(finalContent, kbHits, intent, [...new Set(kbSources)]);
|
||||
|
||||
// 三级升级 + 会话状态
|
||||
let escalationLevel = 0;
|
||||
let escalationReason = null;
|
||||
try {
|
||||
const latestState = (leadStore && await leadStore.get(conv.uuid)) || {};
|
||||
const state = {
|
||||
...latestState,
|
||||
messagesProcessed: (Number(latestState.messagesProcessed) || 0) + 1,
|
||||
consecutiveFails: Number(latestState.consecutiveFails) || 0,
|
||||
};
|
||||
const topScore = kbHits.length ? Number(kbHits[0].score) || 0 : 0;
|
||||
const kbConfidence = topScore >= 0.7 ? 'high' : topScore >= 0.55 ? 'medium' : topScore > 0 ? 'low' : 'none';
|
||||
const decision = escalation.shouldEscalate(
|
||||
latestState,
|
||||
intent,
|
||||
kbConfidence,
|
||||
Number(latestState.consecutiveFails) || 0,
|
||||
kbHits.length
|
||||
);
|
||||
escalationLevel = decision.level;
|
||||
prom.inc('bridge_ai_escalations_total', { level: decision.level });
|
||||
escalationReason = decision.reason;
|
||||
if (decision.level === 1 && !/handoff|human|人工/i.test(finalContent)) {
|
||||
finalContent = escalation.appendEscalationOption(finalContent, currentText);
|
||||
} else if (decision.level === 2) {
|
||||
await escalation.escalateToHuman({ convUuid: conv.uuid, trigger: decision.reason, history: messages.slice(-6) });
|
||||
const zh = /[\u4e00-\u9fff]/.test(currentText);
|
||||
const line = zh ? '\n\n已为您转接人工同事，请稍候。' : '\n\nI have transferred you to a human colleague.';
|
||||
if (!finalContent.includes(line.trim())) finalContent += line;
|
||||
}
|
||||
state.consecutiveFails = (kbConfidence === 'low' || kbConfidence === 'none' || citation.hallucination_flag === true) ? state.consecutiveFails + 1 : 0;
|
||||
state.kbConfidence = kbConfidence;
|
||||
if (leadStore && leadStore.saveState) await leadStore.saveState(conv.uuid, state);
|
||||
else if (leadStore && leadStore.save) await leadStore.save(conv.uuid, state.slots || {});
|
||||
} catch (err) {
|
||||
console.error(JSON.stringify({ ts: new Date().toISOString(), warn: `escalation failed: ${err.message}` }));
|
||||
}
|
||||
|
||||
const alreadyHtml = /<(p|br|a|ul|ol|strong|div)\b/i.test(finalContent);
|
||||
const outboundContent = alreadyHtml ? finalContent : markdownToHtml(finalContent);
|
||||
await client.sendMessage(conv.uuid, outboundContent, { echoId: `bridge-ai-${webhookMessage.uuid}` });
|
||||
@@ -284,8 +368,40 @@ function createAgent(cfg, deps = {}) {
|
||||
reply: finalContent,
|
||||
});
|
||||
}
|
||||
if (citation.hallucination_flag === true || (citation.citation_score !== null && citation.citation_score < 0.5)) {
|
||||
try {
|
||||
const sampleId = await flywheel.captureSample({
|
||||
convId: conv.uuid,
|
||||
query: currentText,
|
||||
reply: finalContent,
|
||||
intent,
|
||||
kbHits: kbHits.slice(0, 10),
|
||||
signal: citation.hallucination_flag === true ? 'hallucination' : 'low_citation',
|
||||
citationScore: citation.citation_score,
|
||||
});
|
||||
if (sampleId) {
|
||||
setImmediate(async () => {
|
||||
try {
|
||||
const review = await flywheel.preLabel({ query: currentText, reply: finalContent, kbHits });
|
||||
if (review) await flywheel.updateReview(sampleId, review);
|
||||
} catch { /* noop */ }
|
||||
});
|
||||
}
|
||||
await client.sendPrivateNote(conv.uuid, `[⚠️ Citation Alert] score=${citation.citation_score}, severity=${citation.severity || 'low'}. This reply may need review.`).catch(() => {});
|
||||
console.log(JSON.stringify({ ts: new Date().toISOString(), agent: 'citation_alert', conversation: conv.uuid, citation_score: citation.citation_score, severity: citation.severity }));
|
||||
} catch (err) {
|
||||
console.error(JSON.stringify({ ts: new Date().toISOString(), warn: `citation/flywheel failed: ${err.message}` }));
|
||||
}
|
||||
}
|
||||
|
||||
const audit = auditAnswer({ finalContent, toolResults: stats.toolResults, toolCalls: stats.toolCalls });
|
||||
audit.injection_level = injection.level;
|
||||
audit.escalation_level = escalationLevel;
|
||||
audit.escalation_reason = escalationReason;
|
||||
audit.experiment = { id: variant.experiment_id, variant: variant.variant };
|
||||
audit.citation_score = citation.citation_score;
|
||||
audit.hallucination_flag = citation.hallucination_flag;
|
||||
audit.uncovered_entities = citation.uncovered_entities;
|
||||
audit.lead_followup = leadFollowup;
|
||||
audit.output_guard = { blocked: guard.blocked, rewritten: guard.rewritten, reasons: guard.reasons, removed_urls: guard.removed_urls, evidence_missing: guard.evidence_missing };
|
||||
await metrics.record({
|
||||
@@ -303,6 +419,7 @@ function createAgent(cfg, deps = {}) {
|
||||
audit,
|
||||
replyLength: finalContent.length,
|
||||
});
|
||||
prom.inc('bridge_ai_runs_total');
|
||||
console.log(JSON.stringify({ ts: new Date().toISOString(), agent: 'replied', conversation: conv.uuid, length: finalContent.length, audit }));
|
||||
return finalContent;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
'use strict';
|
||||
|
||||
function stripHtml(value) {
|
||||
return String(value || '')
|
||||
.replace(/<br\s*\/?>/gi, '\n')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
// 把完整会话记录作为 Note 挂到商机（旧 bridge crm/chatLogger 的 LibreDesk 版）。
|
||||
async function attachChatToOpportunity({ client, twenty, convUuid, opportunityId, maxMessages = 100 }) {
|
||||
if (!client || !twenty || !convUuid || !opportunityId) return null;
|
||||
const messages = await client.getMessages(convUuid, maxMessages);
|
||||
const lines = (messages || [])
|
||||
.filter((m) => m && !m.private && (m.text_content || m.content))
|
||||
.sort((a, b) => Number(a.id) - Number(b.id))
|
||||
.map((m) => {
|
||||
const role = m.sender_type === 'contact' ? 'Customer' : 'Agent';
|
||||
return `[${role}] ${stripHtml(m.text_content || m.content).slice(0, 300)}`;
|
||||
})
|
||||
.filter((line) => line.trim().length > 12);
|
||||
if (!lines.length) return null;
|
||||
const text = lines.join('\n\n---\n\n').slice(0, 20000);
|
||||
const noteId = await twenty.createNote(text);
|
||||
if (noteId) await twenty.linkNoteToOpportunity(noteId, opportunityId);
|
||||
console.log(JSON.stringify({ ts: new Date().toISOString(), crm: 'chat_note', conversation: convUuid, opportunity: opportunityId, note_id: noteId }));
|
||||
return noteId;
|
||||
}
|
||||
|
||||
module.exports = { attachChatToOpportunity, stripHtml };
|
||||
@@ -0,0 +1,314 @@
|
||||
// citation_checker.js — Entity extraction + coverage calculation + hallucination flagging
|
||||
// Shared between offline eval and online scoring paths
|
||||
|
||||
const EU_COUNTRIES = new Set([
|
||||
'austria', 'belgium', 'bulgaria', 'croatia', 'cyprus', 'czech republic', 'czechia',
|
||||
'denmark', 'estonia', 'finland', 'france', 'germany', 'greece', 'hungary', 'ireland',
|
||||
'italy', 'latvia', 'lithuania', 'luxembourg', 'malta', 'netherlands', 'poland',
|
||||
'portugal', 'romania', 'slovakia', 'slovenia', 'spain', 'sweden',
|
||||
// Also common forms / abbreviations
|
||||
'uk', 'united kingdom', 'england', 'scotland', 'wales', 'northern ireland',
|
||||
// German names
|
||||
'deutschland', 'österreich', 'schweiz',
|
||||
// French names
|
||||
'allemagne', 'pays-bas', 'espagne', 'italie', 'belgique',
|
||||
// Dutch names
|
||||
'duitsland', 'nederland', 'belgië', 'spanje', 'italië', 'frankrijk',
|
||||
]);
|
||||
|
||||
const POLICY_TERMS_EN = [
|
||||
'refund', 'return', 'shipping', 'discount', 'vat', 'membership', 'dropship',
|
||||
'factory-direct', 'pre-order', 'preorder', 'backorder', 'cancellation', 'cancel',
|
||||
'exchange', 'warranty', 'guarantee', 'minimum order', 'bulk order', 'wholesale',
|
||||
'retail', 'reseller', 'business', 'corporate', 'invoice', 'payment', 'delivery',
|
||||
'pickup', 'lead time', 'turnaround', 'restock', 'reorder', 'custom', 'personalize',
|
||||
'engrave', 'sample', 'catalog', 'brochure', 'newsletter', 'subscribe', 'unsubscribe',
|
||||
'account', 'register', 'login', 'sign up', 'password',
|
||||
];
|
||||
|
||||
// Multilingual policy term translations (keyed by normalized term)
|
||||
const POLICY_TERMS_I18N = {
|
||||
// German
|
||||
'rückerstattung': 'refund', 'rückgabe': 'return', 'versand': 'shipping',
|
||||
'versandkosten': 'shipping', 'rabatt': 'discount', 'mehrwertsteuer': 'vat',
|
||||
'mitgliedschaft': 'membership', 'stornierung': 'cancellation', 'stornieren': 'cancel',
|
||||
'rechnung': 'invoice', 'zahlung': 'payment', 'lieferung': 'delivery',
|
||||
'lieferzeit': 'delivery', 'großhandel': 'wholesale', 'katalog': 'catalog',
|
||||
'registrieren': 'register', 'konto': 'account',
|
||||
// French
|
||||
'remboursement': 'refund', 'retour': 'return', 'expédition': 'shipping',
|
||||
'frais de livraison': 'shipping', 'livraison': 'delivery', 'remise': 'discount',
|
||||
'réduction': 'discount', 'tva': 'vat', 'adhésion': 'membership',
|
||||
'annulation': 'cancellation', 'annuler': 'cancel', 'facture': 'invoice',
|
||||
'paiement': 'payment', 'grossiste': 'wholesale', 'catalogue': 'catalog',
|
||||
'inscription': 'register', 'compte': 'account',
|
||||
// Dutch
|
||||
'terugbetaling': 'refund', 'retour': 'return', 'retourneren': 'return',
|
||||
'verzending': 'shipping', 'verzendkosten': 'shipping', 'korting': 'discount',
|
||||
'btw': 'vat', 'lidmaatschap': 'membership', 'annulering': 'cancellation',
|
||||
'annuleren': 'cancel', 'factuur': 'invoice', 'betaling': 'payment',
|
||||
'groothandel': 'wholesale', 'catalogus': 'catalog', 'registreren': 'register',
|
||||
'account': 'account',
|
||||
// Italian
|
||||
'rimborso': 'refund', 'reso': 'return', 'spedizione': 'shipping',
|
||||
'costi di spedizione': 'shipping', 'sconto': 'discount', 'iva': 'vat',
|
||||
'iscrizione': 'membership', 'cancellazione': 'cancellation', 'cancellare': 'cancel',
|
||||
'fattura': 'invoice', 'pagamento': 'payment', 'ingrosso': 'wholesale',
|
||||
'catalogo': 'catalog', 'registrare': 'register',
|
||||
};
|
||||
|
||||
// Non-KB intents that should skip hallucination checking
|
||||
const NON_KB_INTENTS = new Set(['product_search', 'order_tracking', 'inventory_check', 'sales_lead']);
|
||||
|
||||
/**
|
||||
* Extract key entities from text.
|
||||
* Entities: amounts (€/EUR), percentages, time limits (days/weeks/months),
|
||||
* country names, policy terms, number+unit combos.
|
||||
*/
|
||||
function extractEntities(text) {
|
||||
if (!text || typeof text !== 'string') return [];
|
||||
const lower = text.toLowerCase();
|
||||
const entities = new Set();
|
||||
|
||||
// Amounts: €200, EUR 200, 200€, 200 euros, 200 eur
|
||||
const amountMatches = text.matchAll(/(?:€|EUR\s?|euros?\s?)\s?(\d+(?:[.,]\d+)?)/gi);
|
||||
for (const m of amountMatches) {
|
||||
entities.add(`amount:${m[1].replace(',', '.')}`);
|
||||
}
|
||||
const amountMatches2 = text.matchAll(/(\d+(?:[.,]\d+)?)\s?(?:€|EUR|euros?|eur)/gi);
|
||||
for (const m of amountMatches2) {
|
||||
entities.add(`amount:${m[1].replace(',', '.')}`);
|
||||
}
|
||||
|
||||
// Percentages: 10%, 10 percent
|
||||
const pctMatches = text.matchAll(/(\d+(?:[.,]\d+)?)\s*(?:%|percent)/gi);
|
||||
for (const m of pctMatches) {
|
||||
entities.add(`pct:${m[1].replace(',', '.')}`);
|
||||
}
|
||||
|
||||
// Time limits: 30 days, 2 weeks, 3 months, 14 Tage, 30 jours, 30 dagen, 5-7 werkdagen
|
||||
const timeMatches = text.matchAll(/(\d+(?:\s*[-–]\s*\d+)?)\s*(?:days?|weeks?|months?|werktage?|werkdagen?|jours?|giorni?|dagen?|Tage?|settimane?|mois)/gi);
|
||||
for (const m of timeMatches) {
|
||||
entities.add(`time:${m[0].toLowerCase().replace(/\s+/g, '')}`);
|
||||
}
|
||||
|
||||
// Number + unit combos: 5kg, 3 pieces, 10 items, 2 kg, 5 stück, 3 pièces, 4 stuks
|
||||
const unitMatches = text.matchAll(/(\d+(?:[.,]\d+)?)\s*(?:kg|g|lb|oz|pieces?|items?|units?|stück|pièces?|stuks?|pairs?|sets?|pcs)/gi);
|
||||
for (const m of unitMatches) {
|
||||
entities.add(`unit:${m[0].toLowerCase().replace(/\s+/g, '')}`);
|
||||
}
|
||||
|
||||
// EU country names
|
||||
const words = lower.split(/[\s,.;:!?"'()\-/]+/).filter(Boolean);
|
||||
// Also check bigrams for "united kingdom", "czech republic"
|
||||
for (let i = 0; i < words.length; i++) {
|
||||
if (EU_COUNTRIES.has(words[i])) {
|
||||
entities.add(`country:${words[i]}`);
|
||||
}
|
||||
if (i + 1 < words.length) {
|
||||
const bigram = words[i] + ' ' + words[i + 1];
|
||||
if (EU_COUNTRIES.has(bigram)) {
|
||||
entities.add(`country:${bigram}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// English policy terms (whole word match)
|
||||
for (const term of POLICY_TERMS_EN) {
|
||||
const pattern = new RegExp('\\b' + term.replace(/[- ]/g, '[- ]') + '\\b', 'i');
|
||||
if (pattern.test(text)) {
|
||||
entities.add(`policy:${term.toLowerCase()}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Multilingual policy terms
|
||||
for (const [foreignTerm, englishTerm] of Object.entries(POLICY_TERMS_I18N)) {
|
||||
if (lower.includes(foreignTerm.toLowerCase())) {
|
||||
entities.add(`policy:${englishTerm}`);
|
||||
}
|
||||
}
|
||||
|
||||
return [...entities];
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize entities for cross-language comparison.
|
||||
* Maps multilingual entities to their English equivalents.
|
||||
*/
|
||||
function normalizeEntity(entity) {
|
||||
// Already prefixed with type: e.g., "policy:refund"
|
||||
return entity;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if entities from the reply are covered by KB hit texts.
|
||||
* Returns { covered, uncovered } entity arrays.
|
||||
*/
|
||||
function computeCoverage(replyEntities, kbHitTexts) {
|
||||
if (!replyEntities.length) return { covered: [], uncovered: [] };
|
||||
|
||||
// Collect all entities from KB texts
|
||||
const kbEntities = new Set();
|
||||
for (const hit of kbHitTexts) {
|
||||
const text = typeof hit === 'string' ? hit : [hit.question, hit.answer, hit.category].filter(Boolean).join(' ');
|
||||
const extracted = extractEntities(text);
|
||||
for (const e of extracted) {
|
||||
kbEntities.add(e);
|
||||
}
|
||||
}
|
||||
|
||||
const covered = [];
|
||||
const uncovered = [];
|
||||
for (const entity of replyEntities) {
|
||||
if (kbEntities.has(entity)) {
|
||||
covered.push(entity);
|
||||
} else {
|
||||
uncovered.push(entity);
|
||||
}
|
||||
}
|
||||
|
||||
return { covered, uncovered };
|
||||
}
|
||||
|
||||
/**
|
||||
* Check for fabricated citations — URLs in the reply that are NOT in the KB sources.
|
||||
* Returns array of fabricated URLs found.
|
||||
*/
|
||||
function detectFabricatedCitations(replyText, kbSources) {
|
||||
if (!replyText || !kbSources || !kbSources.length) return [];
|
||||
|
||||
// Extract URLs from reply
|
||||
const urlMatches = replyText.matchAll(/https?:\/\/[^\s)\]>"']+/gi);
|
||||
const replyUrls = new Set();
|
||||
for (const m of urlMatches) {
|
||||
replyUrls.add(m[0].toLowerCase().replace(/\/+$/, ''));
|
||||
}
|
||||
|
||||
// Normalize KB source URLs
|
||||
const kbUrls = new Set();
|
||||
for (const url of kbSources) {
|
||||
if (typeof url === 'string') {
|
||||
kbUrls.add(url.toLowerCase().replace(/\/+$/, ''));
|
||||
}
|
||||
}
|
||||
|
||||
// Fabricated = URLs in reply not found in KB sources.
|
||||
// 允许"与 KB 来源同源(根域名)"的链接——AI 附加的站内导航链接(如 dropshipping.yehwang.com)
|
||||
// 不算编造引用；只有引用了一个完全不同的域名/来源才标记。
|
||||
const kbOrigins = new Set();
|
||||
for (const url of kbUrls) {
|
||||
try {
|
||||
kbOrigins.add(new URL(url).origin.toLowerCase());
|
||||
} catch (e) { /* 忽略非法 URL */ }
|
||||
}
|
||||
const fabricated = [];
|
||||
for (const url of replyUrls) {
|
||||
if (kbUrls.has(url)) continue;
|
||||
try {
|
||||
const origin = new URL(url).origin.toLowerCase();
|
||||
if (kbOrigins.has(origin)) continue; // 同源站内链接 → 不算编造
|
||||
} catch (e) { /* 非法 URL 跳过 */ }
|
||||
fabricated.push(url);
|
||||
}
|
||||
|
||||
return fabricated;
|
||||
}
|
||||
|
||||
/**
|
||||
* Main entry point: check citation consistency of a reply against KB hits.
|
||||
*
|
||||
* @param {string} replyText - Agent's final reply text (stripped of HTML)
|
||||
* @param {Array<string|object>} kbHitTexts - KB hit texts (strings or objects with question/answer/category)
|
||||
* @param {string} intent - Current intent label
|
||||
* @param {Array<string>} [kbSources] - KB source URLs (for fabricated citation detection)
|
||||
* @returns {{ citation_score: number|null, hallucination_flag: boolean|null, extracted_entities: string[], covered_entities: string[], uncovered_entities: string[], severity: string|null }}
|
||||
*/
|
||||
function checkCitation(replyText, kbHitTexts, intent, kbSources) {
|
||||
// Non-KB intents: skip hallucination checking
|
||||
if (NON_KB_INTENTS.has(intent)) {
|
||||
return {
|
||||
citation_score: null,
|
||||
hallucination_flag: null,
|
||||
extracted_entities: [],
|
||||
covered_entities: [],
|
||||
uncovered_entities: [],
|
||||
severity: null,
|
||||
};
|
||||
}
|
||||
|
||||
// No reply text: nothing to check
|
||||
if (!replyText || typeof replyText !== 'string') {
|
||||
return {
|
||||
citation_score: null,
|
||||
hallucination_flag: null,
|
||||
extracted_entities: [],
|
||||
covered_entities: [],
|
||||
uncovered_entities: [],
|
||||
severity: null,
|
||||
};
|
||||
}
|
||||
|
||||
// Check for fabricated citations first (most severe)
|
||||
const fabricatedUrls = detectFabricatedCitations(replyText, kbSources || []);
|
||||
if (fabricatedUrls.length > 0) {
|
||||
return {
|
||||
citation_score: 0,
|
||||
hallucination_flag: true,
|
||||
extracted_entities: [],
|
||||
covered_entities: [],
|
||||
uncovered_entities: [],
|
||||
severity: 'fabricated_citation',
|
||||
};
|
||||
}
|
||||
|
||||
// No KB hits but KB-intent (policy_faq/general): if reply has substantive entities, flag
|
||||
const hasKbHits = kbHitTexts && kbHitTexts.length > 0;
|
||||
const extractedEntities = extractEntities(replyText);
|
||||
|
||||
if (!hasKbHits) {
|
||||
// No KB data to check against — if there are policy entities, likely hallucinated
|
||||
const hasPolicyEntities = extractedEntities.some(e => e.startsWith('policy:'));
|
||||
const hasFactualEntities = extractedEntities.some(e => e.startsWith('amount:') || e.startsWith('pct:') || e.startsWith('time:'));
|
||||
|
||||
if (hasPolicyEntities || hasFactualEntities) {
|
||||
return {
|
||||
citation_score: 0,
|
||||
hallucination_flag: true,
|
||||
extracted_entities: extractedEntities,
|
||||
covered_entities: [],
|
||||
uncovered_entities: extractedEntities,
|
||||
severity: 'no_kb_support',
|
||||
};
|
||||
}
|
||||
|
||||
// No entities extracted, no KB hits — can't determine, not flagged
|
||||
return {
|
||||
citation_score: null,
|
||||
hallucination_flag: null,
|
||||
extracted_entities: [],
|
||||
covered_entities: [],
|
||||
uncovered_entities: [],
|
||||
severity: null,
|
||||
};
|
||||
}
|
||||
|
||||
// Normal case: compute coverage
|
||||
const { covered, uncovered } = computeCoverage(extractedEntities, kbHitTexts);
|
||||
const citationScore = extractedEntities.length > 0
|
||||
? covered.length / extractedEntities.length
|
||||
: 1; // No entities to check = nothing to hallucinate about
|
||||
|
||||
// Hallucination threshold: citation_score < 0.5 for KB intents
|
||||
const hallucinationFlag = citationScore < 0.5;
|
||||
|
||||
return {
|
||||
citation_score: Math.round(citationScore * 1000) / 1000, // 3 decimal places
|
||||
hallucination_flag: hallucinationFlag,
|
||||
extracted_entities: extractedEntities,
|
||||
covered_entities: covered,
|
||||
uncovered_entities: uncovered,
|
||||
severity: hallucinationFlag ? 'low_coverage' : null,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { checkCitation, extractEntities, computeCoverage, detectFabricatedCitations };
|
||||
@@ -0,0 +1,65 @@
|
||||
'use strict';
|
||||
|
||||
// 三级升级策略（旧 bridge escalation/manager 的 LibreDesk 版）：
|
||||
// Level 0 正常；Level 1 追加“是否转人工”；Level 2 私聊备注 + 分配人工。
|
||||
function createEscalation({ client, handoffUserId = 1 }) {
|
||||
function shouldEscalate(state, intent, kbConfidence, consecutiveFails, kbHitCount = 0) {
|
||||
const processed = Number(state && state.messagesProcessed) || 0;
|
||||
if (consecutiveFails >= 3) return { level: 2, reason: 'consecutive_fails' };
|
||||
if (processed > 8 && !(state && state.opportunityId)) return { level: 2, reason: 'long_no_lead' };
|
||||
if (intent === 'policy_faq' && kbConfidence === 'low' && kbHitCount === 0) return { level: 1, reason: 'low_kb_confidence' };
|
||||
if (kbConfidence === 'none' && !['product_search', 'sales_lead', 'order_tracking', 'inventory_check'].includes(intent)) return { level: 1, reason: 'no_kb_match' };
|
||||
if (processed > 5 && consecutiveFails >= 2) return { level: 1, reason: 'repeated_low_confidence' };
|
||||
return { level: 0, reason: null };
|
||||
}
|
||||
|
||||
function appendEscalationOption(reply, query) {
|
||||
const text = String(reply || '').replace(/\s+$/, '');
|
||||
const hints = {
|
||||
zh: '您想转人工客服吗？',
|
||||
de: 'Möchten Sie mit einem Mitarbeiter sprechen?',
|
||||
fr: 'Souhaitez-vous parler à un conseiller ?',
|
||||
it: 'Vuole parlare con un operatore?',
|
||||
nl: 'Wilt u met een medewerker spreken?',
|
||||
};
|
||||
let lang = null;
|
||||
if (/[\u4e00-\u9fff]/.test(query)) lang = 'zh';
|
||||
else if (/[äöüß]/i.test(query)) lang = 'de';
|
||||
else if (/[éèêëàâùûîïôç]/i.test(query)) lang = 'fr';
|
||||
else if (/\b(hoe|voor|wilt|bent|geen|verzending|levering|bestelling)\b/i.test(query)) lang = 'nl';
|
||||
const option = lang ? `\n\n${hints[lang]} / Would you like to speak to a human agent?` : '\n\nWould you like to speak to a human agent?';
|
||||
return text + option;
|
||||
}
|
||||
|
||||
async function escalateToHuman({ convUuid, trigger, history }) {
|
||||
const lines = [`Escalation trigger: ${trigger || 'unknown'}`];
|
||||
if (history && history.length) {
|
||||
lines.push('Recent context:');
|
||||
for (const m of history.slice(-4)) {
|
||||
const role = m.role === 'user' ? 'Customer' : 'AI';
|
||||
lines.push(` [${role}] ${String(m.content || '').slice(0, 150)}`);
|
||||
}
|
||||
}
|
||||
try {
|
||||
await client.sendPrivateNote(convUuid, lines.join('\n'));
|
||||
await client.assignUser(convUuid, handoffUserId);
|
||||
return true;
|
||||
} catch (err) {
|
||||
console.error(JSON.stringify({ ts: new Date().toISOString(), warn: `escalate failed: ${err.message}` }));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function applyEmotionStrategy(emotion) {
|
||||
switch (emotion) {
|
||||
case 'angry': return { skipAI: false, escalate: true, maxToolTurns: 2, tone: 'empathetic' };
|
||||
case 'frustrated': return { skipAI: false, escalate: false, maxToolTurns: 3, tone: 'patient' };
|
||||
case 'urgent': return { skipAI: false, escalate: false, maxToolTurns: 3, tone: 'direct' };
|
||||
default: return { skipAI: false, escalate: false, maxToolTurns: 5, tone: 'normal' };
|
||||
}
|
||||
}
|
||||
|
||||
return { shouldEscalate, appendEscalationOption, escalateToHuman, applyEmotionStrategy };
|
||||
}
|
||||
|
||||
module.exports = { createEscalation };
|
||||
@@ -0,0 +1,38 @@
|
||||
'use strict';
|
||||
|
||||
const crypto = require('crypto');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
// A/B 实验：按 conversationId 稳定分桶（旧 bridge config/experiments.js 精简版）。
|
||||
function createExperimentManager(cfg = {}) {
|
||||
const file = cfg.experimentsPath || path.resolve(__dirname, '..', 'config', 'experiments.json');
|
||||
let experiments = [];
|
||||
try {
|
||||
const data = JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
experiments = Array.isArray(data.experiments) ? data.experiments : [];
|
||||
} catch {
|
||||
experiments = [];
|
||||
}
|
||||
|
||||
function getVariant(convId, experimentId) {
|
||||
const exp = experiments.find((e) => e.active && (!experimentId || e.id === experimentId));
|
||||
if (!exp) return { experiment_id: null, variant: null, overriddenParams: null };
|
||||
const hash = crypto.createHash('md5').update(`${convId}:${exp.id}`).digest('hex');
|
||||
const bucket = parseInt(hash.slice(0, 8), 16) % 100;
|
||||
let cumulative = 0;
|
||||
for (const v of exp.variants || []) {
|
||||
cumulative += v.weight || 50;
|
||||
if (bucket < cumulative) return { experiment_id: exp.id, variant: v.name, overriddenParams: v.overriddenParams || null };
|
||||
}
|
||||
return { experiment_id: exp.id, variant: (exp.variants && exp.variants[0] && exp.variants[0].name) || null, overriddenParams: null };
|
||||
}
|
||||
|
||||
function getActiveExperiments() {
|
||||
return experiments.filter((e) => e.active);
|
||||
}
|
||||
|
||||
return { getVariant, getActiveExperiments };
|
||||
}
|
||||
|
||||
module.exports = { createExperimentManager };
|
||||
@@ -0,0 +1,26 @@
|
||||
'use strict';
|
||||
|
||||
// 信息型页面 LLM 提炼成 FAQ（旧 bridge faq/llmRefiner.js 的移植）。
|
||||
function createFaqRefiner(llm) {
|
||||
const SYSTEM = `You turn website page content into a clean FAQ knowledge base entry.
|
||||
Rules: output 3-8 "### Q: ..." / answer pairs based ONLY on the page content; never invent facts, numbers, or policies; answers 1-4 sentences; English; output only the FAQ body, no code fences.`;
|
||||
|
||||
async function refineToFaq(rawMarkdown, category) {
|
||||
const text = String(rawMarkdown || '').slice(0, 6000);
|
||||
if (!text.trim()) return null;
|
||||
const res = await llm.chat({
|
||||
messages: [
|
||||
{ role: 'system', content: SYSTEM },
|
||||
{ role: 'user', content: `Page category: ${category || 'general'}\n\nRaw page content:\n${text}` },
|
||||
],
|
||||
temperature: 0.2,
|
||||
maxTokens: 800,
|
||||
});
|
||||
const faq = String(res.content || '').trim();
|
||||
return faq.includes('### Q:') ? faq : null;
|
||||
}
|
||||
|
||||
return { refineToFaq };
|
||||
}
|
||||
|
||||
module.exports = { createFaqRefiner };
|
||||
@@ -0,0 +1,142 @@
|
||||
// middleware/flywheel_writer.js — 数据飞轮: 线上失败样本捕获 + LLM 预标注
|
||||
// 捕获幻觉/低分/升级/注入等可疑样本(query+reply+检索KB), LLM 预标注(疑似原因+
|
||||
// 建议修正), 供人工每周确认后回流修正 KB / golden_dataset(防复发)。
|
||||
// 设计: 捕获失败静默(不影响主链路); 预标注失败不影响样本入库(人工仍可看)。
|
||||
function createFlywheelWriter({ pool, callLLM }) {
|
||||
/** 捕获一个可疑样本(单行 INSERT, 失败静默)。返回 sampleId 或 null。 */
|
||||
let ready = null;
|
||||
function ensureTable() {
|
||||
if (!pool) return Promise.resolve();
|
||||
if (!ready) {
|
||||
ready = pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS flywheel_samples (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
conv_id TEXT,
|
||||
query TEXT,
|
||||
reply TEXT,
|
||||
intent TEXT,
|
||||
kb_hits JSONB,
|
||||
signal TEXT DEFAULT 'hallucination',
|
||||
citation_score REAL,
|
||||
llm_review JSONB,
|
||||
status TEXT DEFAULT 'pending',
|
||||
note TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`).catch((e) => console.error('[flywheel] table init error:', e.message.slice(0, 120)));
|
||||
}
|
||||
return ready;
|
||||
}
|
||||
|
||||
async function captureSample({ convId, query, reply, intent, kbHits, signal, citationScore }) {
|
||||
if (!pool) return null;
|
||||
try {
|
||||
await ensureTable();
|
||||
const { rows } = await pool.query(
|
||||
`INSERT INTO flywheel_samples (conv_id, query, reply, intent, kb_hits, signal, citation_score)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id`,
|
||||
[
|
||||
convId !== undefined ? convId : null,
|
||||
(query || '').slice(0, 2000),
|
||||
(reply || '').slice(0, 4000),
|
||||
intent || null,
|
||||
kbHits && kbHits.length ? JSON.stringify(kbHits.slice(0, 10)) : null,
|
||||
signal || 'hallucination',
|
||||
citationScore !== undefined ? citationScore : null,
|
||||
]
|
||||
);
|
||||
return rows[0] ? rows[0].id : null;
|
||||
} catch (e) {
|
||||
console.error('[flywheel] capture error:', e.message.slice(0, 120));
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* LLM 预标注: 判断回复是否与检索到的 KB 内容矛盾/编造数字, 给出建议修正。
|
||||
* 失败返回 null(样本仍入库, 人工看)。
|
||||
*/
|
||||
async function preLabel({ query, reply, kbHits }) {
|
||||
if (!callLLM) return null;
|
||||
try {
|
||||
const kbText = (kbHits || []).slice(0, 5).map((h) => `KB#${h.id}: ${h.question}\n${h.answer}`).join('\n---\n');
|
||||
const res = await callLLM(
|
||||
[{ role: 'user', content: `You are reviewing an AI customer-service reply for factual errors.
|
||||
Compare the reply against the retrieved knowledge base (KB) entries below.
|
||||
If the reply contains numbers/facts (amounts, percentages, time limits, thresholds) NOT present in the KB text, or contradicts the KB, mark suspected=true and give the reason + suggested correct answer (from the KB if available).
|
||||
Output JSON only: {"suspected": true/false, "reason": "...", "suggested_fix": "...", "kb_ref": "KB#<id> or null"}
|
||||
|
||||
KB:
|
||||
${kbText.slice(0, 3000)}
|
||||
|
||||
REPLY:
|
||||
${(reply || '').slice(0, 1500)}
|
||||
|
||||
QUERY:
|
||||
${(query || '').slice(0, 500)}
|
||||
|
||||
JSON:` }],
|
||||
'You are a careful fact-checker. Output ONLY valid JSON.', 300, null
|
||||
);
|
||||
const m = (res.text || '').match(/\{[\s\S]*\}/);
|
||||
if (!m) return null;
|
||||
const parsed = JSON.parse(m[0]);
|
||||
return {
|
||||
suspected: parsed.suspected === true,
|
||||
reason: parsed.reason || null,
|
||||
suggested_fix: parsed.suggested_fix || null,
|
||||
kb_ref: parsed.kb_ref || null,
|
||||
};
|
||||
} catch (e) {
|
||||
console.error('[flywheel] preLabel error:', e.message.slice(0, 100));
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** 写预标注结果(LLM 预标注完成后回填)。 */
|
||||
async function updateReview(sampleId, review) {
|
||||
if (!pool || !sampleId || !review) return;
|
||||
try {
|
||||
await pool.query(`UPDATE flywheel_samples SET llm_review = $1 WHERE id = $2`, [JSON.stringify(review), sampleId]);
|
||||
} catch (e) {
|
||||
console.error('[flywheel] updateReview error:', e.message.slice(0, 100));
|
||||
}
|
||||
}
|
||||
|
||||
/** 待标注列表(分页)。返回 { samples, total }。 */
|
||||
async function listSamples({ status = 'pending', limit = 50, offset = 0 } = {}) {
|
||||
if (!pool) return { samples: [], total: 0 };
|
||||
try {
|
||||
const { rows } = await pool.query(
|
||||
`SELECT id, conv_id, query, reply, intent, kb_hits, signal, citation_score, llm_review, status, note, created_at
|
||||
FROM flywheel_samples
|
||||
WHERE status = $1
|
||||
ORDER BY created_at DESC
|
||||
LIMIT $2 OFFSET $3`, [status, limit, offset]
|
||||
);
|
||||
const { rows: [cnt] } = await pool.query(
|
||||
`SELECT COUNT(*)::int AS total FROM flywheel_samples WHERE status = $1`, [status]
|
||||
);
|
||||
return { samples: rows, total: cnt.total };
|
||||
} catch (e) {
|
||||
console.error('[flywheel] listSamples error:', e.message.slice(0, 100));
|
||||
return { samples: [], total: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
/** 标注样本(人工确认/驳回)。 */
|
||||
async function updateSample(id, { status, note }) {
|
||||
if (!pool || !id) return false;
|
||||
try {
|
||||
await pool.query(`UPDATE flywheel_samples SET status = $1, note = $2 WHERE id = $3`, [status || 'skip', note || null, id]);
|
||||
return true;
|
||||
} catch (e) {
|
||||
console.error('[flywheel] updateSample error:', e.message.slice(0, 100));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return { captureSample, preLabel, updateReview, listSamples, updateSample };
|
||||
}
|
||||
|
||||
module.exports = { createFlywheelWriter };
|
||||
@@ -0,0 +1,51 @@
|
||||
'use strict';
|
||||
|
||||
// KB 版本查看/回滚：备份表命名 kb_docs_v<timestamp>，当前表为 kb_docs。
|
||||
function quoteIdent(name) {
|
||||
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) throw new Error(`invalid identifier: ${name}`);
|
||||
return `"${name}"`;
|
||||
}
|
||||
|
||||
async function listVersions(pool) {
|
||||
const backups = await pool.query(
|
||||
`SELECT tablename FROM pg_tables WHERE schemaname='public' AND tablename LIKE 'kb_docs_v%' ORDER BY tablename DESC`
|
||||
);
|
||||
const runs = await pool.query('SELECT * FROM kb_sync_runs ORDER BY id DESC LIMIT 20').catch(() => ({ rows: [] }));
|
||||
return { backups: backups.rows.map((r) => r.tablename), runs: runs.rows };
|
||||
}
|
||||
|
||||
async function rollback(pool, target) {
|
||||
if (!/^kb_docs_v[a-zA-Z0-9_]+$/.test(String(target))) throw new Error(`invalid backup table: ${target}`);
|
||||
const exists = await pool.query('SELECT to_regclass($1) IS NOT NULL AS ok', [`public.${target}`]);
|
||||
if (!exists.rows[0].ok) throw new Error(`backup not found: ${target}`);
|
||||
const hasCurrent = await pool.query("SELECT to_regclass('public.kb_docs') IS NOT NULL AS ok");
|
||||
const stamp = new Date().toISOString().replace(/[-:TZ.]/g, '').slice(0, 14);
|
||||
const currentBackup = `kb_docs_before_rollback_${stamp}`;
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
if (hasCurrent.rows[0].ok) {
|
||||
const idx = await client.query("SELECT indexname FROM pg_indexes WHERE schemaname='public' AND tablename='kb_docs'");
|
||||
await client.query(`ALTER TABLE kb_docs RENAME TO ${quoteIdent(currentBackup)}`);
|
||||
for (const row of idx.rows) {
|
||||
const newName = `${currentBackup}_${row.indexname.replace(/^kb_docs_/, '').replace(/_idx$/, '')}_idx`;
|
||||
await client.query(`ALTER INDEX ${quoteIdent(row.indexname)} RENAME TO ${quoteIdent(newName)}`).catch(() => {});
|
||||
}
|
||||
}
|
||||
await client.query(`ALTER TABLE ${quoteIdent(target)} RENAME TO kb_docs`);
|
||||
const targetIdx = await client.query(`SELECT indexname FROM pg_indexes WHERE schemaname='public' AND tablename='kb_docs'`);
|
||||
for (const row of targetIdx.rows) {
|
||||
const suffix = row.indexname.includes('vec') ? 'vec_idx' : row.indexname.includes('tsv') ? 'tsv_idx' : row.indexname.includes('parent') ? 'parent_idx' : null;
|
||||
if (suffix) await client.query(`ALTER INDEX ${quoteIdent(row.indexname)} RENAME TO ${quoteIdent('kb_docs_' + suffix)}`).catch(() => {});
|
||||
}
|
||||
await client.query('COMMIT');
|
||||
return { rolled_back_to: target, previous: hasCurrent.rows[0].ok ? currentBackup : null };
|
||||
} catch (err) {
|
||||
await client.query('ROLLBACK');
|
||||
throw err;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { listVersions, rollback };
|
||||
@@ -0,0 +1,20 @@
|
||||
'use strict';
|
||||
|
||||
// 语言检测 + 硬性语言约束（最新客户消息决定回复语言）。
|
||||
function detectLanguage(text) {
|
||||
const q = String(text || '');
|
||||
if (/[\u3400-\u9fff]/.test(q)) return 'Chinese';
|
||||
if (/\b(und|nicht|für|mit|bitte|haben|preis|menge|lieferung)\b/i.test(q)) return 'German';
|
||||
if (/\b(et|pour|avec|bonjour|prix|quantité|livraison|merci)\b/i.test(q)) return 'French';
|
||||
if (/\b(e|per|con|prezzo|quantità|spedizione|grazie)\b/i.test(q)) return 'Italian';
|
||||
if (/\b(y|para|con|precio|cantidad|envío|gracias)\b/i.test(q)) return 'Spanish';
|
||||
if (/\b(en|voor|met|prijs|aantal|levering|bedankt)\b/i.test(q)) return 'Dutch';
|
||||
return 'English';
|
||||
}
|
||||
|
||||
function languageMandate(text) {
|
||||
const lang = detectLanguage(text);
|
||||
return `\n\nLANGUAGE MANDATE (hard rule): The customer's latest message is in ${lang}. Reply ENTIRELY in ${lang}, even if earlier messages or tool results are in another language. Do NOT mix languages.`;
|
||||
}
|
||||
|
||||
module.exports = { detectLanguage, languageMandate };
|
||||
@@ -1,6 +1,7 @@
|
||||
'use strict';
|
||||
|
||||
const crypto = require('crypto');
|
||||
const prom = require('./prom');
|
||||
|
||||
// Phase 0：验签 -> 去重 -> 只处理 contact 的 message.created -> 回一条占位消息。
|
||||
// Phase 1 会把这里替换成 bridge AI 编排。
|
||||
@@ -29,6 +30,7 @@ function createLibreDeskWebhook(cfg, deps = {}) {
|
||||
async function handle(body) {
|
||||
if (!body || body.event !== 'message.created') return;
|
||||
const msg = body.payload || {};
|
||||
prom.inc('bridge_webhook_events_total', { event: body.event || 'unknown' });
|
||||
if (msg.sender_type !== 'contact' || msg.private) return;
|
||||
if (!msg.uuid || !msg.conversation_uuid) return;
|
||||
if (!markSeen(msg.uuid)) return;
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
'use strict';
|
||||
|
||||
const { ToolError } = require('./errors');
|
||||
const { attachChatToOpportunity } = require('./chat-log');
|
||||
|
||||
const STAGES = new Set(['NEW', 'SCREENING', 'MEETING', 'PROPOSAL', 'CUSTOMER']);
|
||||
|
||||
// 对齐旧 bridge leadPipeline：指纹去重、跨渠道复用/同步、阶段推进、CRM 故障 fail-closed。
|
||||
function createOpportunityService({ cfg, twenty, leadStore }) {
|
||||
function createOpportunityService({ cfg, twenty, leadStore, client }) {
|
||||
function splitName(full) {
|
||||
const parts = String(full || '').trim().split(/[\s.]+/).filter(Boolean);
|
||||
if (parts.length >= 2) return { firstName: parts[0], lastName: parts.slice(1).join(' ') };
|
||||
@@ -140,6 +141,7 @@ function createOpportunityService({ cfg, twenty, leadStore }) {
|
||||
}
|
||||
const noteId = await twenty.createNote(summary);
|
||||
if (noteId) await twenty.linkNoteToOpportunity(noteId, existing.id);
|
||||
setImmediate(() => attachChatToOpportunity({ client, twenty, convUuid: conversationUuid, opportunityId: existing.id }).catch(() => {}));
|
||||
const stage = await advanceStage({ ctx, state, slots, email: slots.email || email });
|
||||
return {
|
||||
created: false,
|
||||
@@ -187,6 +189,7 @@ function createOpportunityService({ cfg, twenty, leadStore }) {
|
||||
if (noteId) await twenty.linkNoteToOpportunity(noteId, opp.id);
|
||||
const dueAt = new Date(Date.now() + 3 * 86400 * 1000).toISOString();
|
||||
const taskId = await twenty.createTask(`Follow up: ${slots.company}`, summary, dueAt, opp.id);
|
||||
setImmediate(() => attachChatToOpportunity({ client, twenty, convUuid: conversationUuid, opportunityId: opp.id }).catch(() => {}));
|
||||
const stage = await advanceStage({ ctx, state, slots, email: slots.email || email });
|
||||
|
||||
return {
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
'use strict';
|
||||
|
||||
// 极简 Prometheus 指标注册表（无外部依赖）。
|
||||
const counters = new Map();
|
||||
const startedAt = Date.now();
|
||||
|
||||
function key(name, labels = {}) {
|
||||
const sorted = Object.keys(labels).sort().map((k) => `${k}="${String(labels[k]).replace(/["\\]/g, '')}"`).join(',');
|
||||
return sorted ? `${name}{${sorted}}` : name;
|
||||
}
|
||||
|
||||
function inc(name, labels = {}, value = 1) {
|
||||
const k = key(name, labels);
|
||||
counters.set(k, (counters.get(k) || 0) + Number(value || 1));
|
||||
}
|
||||
|
||||
function render() {
|
||||
const lines = [
|
||||
'# HELP bridge_uptime_seconds Process uptime in seconds',
|
||||
'# TYPE bridge_uptime_seconds gauge',
|
||||
`bridge_uptime_seconds ${((Date.now() - startedAt) / 1000).toFixed(1)}`,
|
||||
'# HELP bridge_process_memory_bytes Process RSS bytes',
|
||||
'# TYPE bridge_process_memory_bytes gauge',
|
||||
`bridge_process_memory_bytes ${process.memoryUsage().rss}`,
|
||||
];
|
||||
for (const [k, v] of counters) {
|
||||
const name = k.split('{')[0];
|
||||
lines.push(`# TYPE ${name} counter`);
|
||||
lines.push(`${k} ${v}`);
|
||||
}
|
||||
return lines.join('\n') + '\n';
|
||||
}
|
||||
|
||||
module.exports = { inc, render };
|
||||
@@ -0,0 +1,22 @@
|
||||
'use strict';
|
||||
|
||||
// LLM rerank：对 RAG 候选做二次排序（旧 bridge rag/reranker.js 的移植）。
|
||||
function createReranker(llm) {
|
||||
async function rerank(query, candidates, topK = 3) {
|
||||
if (!candidates || candidates.length <= topK) return candidates || [];
|
||||
const docs = candidates.map((c, i) => `[${i}] (category: ${c.category || ''}) Q: ${c.question}\nA: ${String(c.answer || '').slice(0, 200)}`).join('\n\n');
|
||||
const prompt = `Rank these knowledge base entries by relevance to the query. Output ONLY the indices of the ${topK} most relevant entries, in order, comma-separated. Example: 3,0,7\n\nQuery: "${query}"\n\nEntries:\n${docs}`;
|
||||
try {
|
||||
const res = await llm.chat({ messages: [{ role: 'user', content: prompt }], temperature: 0, maxTokens: 50 });
|
||||
const indices = String(res.content || '').trim().split(',').map((s) => parseInt(s.trim(), 10)).filter((i) => Number.isInteger(i) && i >= 0 && i < candidates.length);
|
||||
if (!indices.length) return candidates.slice(0, topK);
|
||||
return indices.slice(0, topK).map((i) => candidates[i]);
|
||||
} catch (err) {
|
||||
console.error(JSON.stringify({ ts: new Date().toISOString(), warn: `rerank failed: ${err.message}` }));
|
||||
return candidates.slice(0, topK);
|
||||
}
|
||||
}
|
||||
return { rerank };
|
||||
}
|
||||
|
||||
module.exports = { createReranker };
|
||||
@@ -7,6 +7,7 @@ const { createTools, resolveSiteProfile } = require('./tools');
|
||||
const { createIdentity } = require('./identity');
|
||||
const { createLibreDeskWebhook } = require('./libredesk-webhook');
|
||||
const { createAgent } = require('./agent');
|
||||
const prom = require('./prom');
|
||||
const { ToolError } = require('./errors');
|
||||
|
||||
const MAX_BODY_BYTES = 256 * 1024;
|
||||
@@ -113,6 +114,14 @@ function createServer(options = {}) {
|
||||
});
|
||||
}
|
||||
|
||||
// GET /metrics — Prometheus 文本指标（默认仅内网/LAN 暴露）
|
||||
if (req.method === 'GET' && url === '/metrics') {
|
||||
const body = prom.render();
|
||||
res.writeHead(200, { 'content-type': 'text/plain; version=0.0.4; charset=utf-8', 'content-length': Buffer.byteLength(body) });
|
||||
res.end(body);
|
||||
return;
|
||||
}
|
||||
|
||||
// POST /libredesk/webhook — 方案 A：LibreDesk 消息事件进入 bridge（HMAC 验签）
|
||||
if (req.method === 'POST' && url === '/libredesk/webhook') {
|
||||
const raw = await readRawBody(req);
|
||||
|
||||
+5
-3
@@ -325,8 +325,9 @@ function createTools(cfg, deps = {}) {
|
||||
const leadStore = deps.leadStore || createLeadStore(cfg);
|
||||
const retriever = deps.retriever || null;
|
||||
const sitemap = deps.sitemap || createSiteMap(cfg);
|
||||
const opportunityService = deps.opportunityService || createOpportunityService({ cfg, twenty, leadStore });
|
||||
const opportunityService = deps.opportunityService || createOpportunityService({ cfg, twenty, leadStore, client: deps.client });
|
||||
const mailHistory = deps.mailHistory || createMailHistory(cfg);
|
||||
const reranker = deps.reranker || null;
|
||||
function missingLeadFields(slots) {
|
||||
const missing = [];
|
||||
if (!slots.company) missing.push('company');
|
||||
@@ -376,7 +377,8 @@ function createTools(cfg, deps = {}) {
|
||||
const query = String(args.query || '').trim();
|
||||
if (!query) throw new ToolError(400, 'invalid_arguments', 'query is required');
|
||||
const limit = Math.min(Number(args.limit) || 5, 8);
|
||||
const results = await retriever.searchKB(query, limit);
|
||||
let results = await retriever.searchKB(query, Math.max(limit, 8));
|
||||
if (reranker) results = await reranker.rerank(query, results, limit);
|
||||
return {
|
||||
results,
|
||||
count: results.length,
|
||||
@@ -697,7 +699,7 @@ function createTools(cfg, deps = {}) {
|
||||
async function sendVerificationCode(args, ctx) {
|
||||
if (!deps.verification) throw new ToolError(503, 'not_configured', 'Verification is not configured');
|
||||
if (!ctx.conversationUuid) throw new ToolError(400, 'missing_conversation', 'conversation context is required');
|
||||
const out = await deps.verification.sendCode(ctx.conversationUuid, args.email);
|
||||
const out = await deps.verification.sendCode(ctx.conversationUuid, args.email, { language: ctx.language });
|
||||
return { ...out, note: 'Ask the customer to check their inbox and reply with the 6-digit code.' };
|
||||
}
|
||||
|
||||
|
||||
+15
-10
@@ -13,6 +13,17 @@ function normalizeEmail(value) {
|
||||
return String(value || '').trim().toLowerCase();
|
||||
}
|
||||
|
||||
function otpEmail(language, code) {
|
||||
const templates = {
|
||||
zh: { subject: 'Yehwang 验证码', text: `您的 Yehwang 验证码是：${code}\n\n10 分钟内有效。如非本人操作请忽略。` },
|
||||
de: { subject: 'Ihr Yehwang Bestätigungscode', text: `Ihr Yehwang Bestätigungscode lautet: ${code}\n\nGültig für 10 Minuten.` },
|
||||
fr: { subject: 'Votre code de vérification Yehwang', text: `Votre code de vérification Yehwang est : ${code}\n\nValable 10 minutes.` },
|
||||
nl: { subject: 'Uw Yehwang verificatiecode', text: `Uw Yehwang verificatiecode is: ${code}\n\n10 minuten geldig.` },
|
||||
en: { subject: 'Your Yehwang verification code', text: `Your Yehwang verification code is: ${code}\n\nIt expires in 10 minutes.` },
|
||||
};
|
||||
return templates[language] || templates.en;
|
||||
}
|
||||
|
||||
function createVerification(cfg, deps = {}) {
|
||||
const memory = new Map();
|
||||
const client = deps.redis || (cfg.redisUrl ? new RedisClient(cfg.redisUrl, { timeoutMs: 3000 }) : null);
|
||||
@@ -40,7 +51,7 @@ function createVerification(cfg, deps = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
async function sendCode(conversationUuid, email) {
|
||||
async function sendCode(conversationUuid, email, opts = {}) {
|
||||
const conv = String(conversationUuid || '');
|
||||
const address = normalizeEmail(email);
|
||||
if (!conv) throw new Error('missing conversation');
|
||||
@@ -55,17 +66,11 @@ function createVerification(cfg, deps = {}) {
|
||||
const pending = JSON.stringify({ code, email: address, attempts: 0, createdAt: Date.now() });
|
||||
await setEx(otpKey(conv), pending, OTP_TTL_SEC);
|
||||
|
||||
const text = [
|
||||
'Your Yehwang verification code is: ' + code,
|
||||
'',
|
||||
'It expires in 10 minutes. If you did not request it, ignore this email.',
|
||||
'',
|
||||
'Yehwang Support',
|
||||
].join('\n');
|
||||
const tpl = otpEmail(opts.language, code);
|
||||
try {
|
||||
await sendMail(
|
||||
{ host: cfg.smtpHost, port: cfg.smtpPort, user: cfg.smtpUser, pass: cfg.smtpPass, from: cfg.smtpFrom },
|
||||
{ to: address, subject: 'Your Yehwang verification code', text }
|
||||
{ to: address, subject: tpl.subject, text: tpl.text }
|
||||
);
|
||||
} catch (err) {
|
||||
await del(otpKey(conv));
|
||||
@@ -107,4 +112,4 @@ function createVerification(cfg, deps = {}) {
|
||||
return { sendCode, verifyCode, getVerifiedEmail };
|
||||
}
|
||||
|
||||
module.exports = { createVerification };
|
||||
module.exports = { createVerification, otpEmail };
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
const { describe, it } = require('node:test');
|
||||
const assert = require('node:assert');
|
||||
const { checkCitation, extractEntities, computeCoverage, detectFabricatedCitations } = require('../src/citation-checker');
|
||||
|
||||
describe('citation_checker', () => {
|
||||
describe('extractEntities', () => {
|
||||
it('extracts EUR amounts with € symbol', () => {
|
||||
const entities = extractEntities('Free shipping for orders over €200.');
|
||||
const amountEntities = entities.filter(e => e.startsWith('amount:'));
|
||||
assert.ok(amountEntities.some(e => e === 'amount:200'), `Expected amount:200, got ${amountEntities}`);
|
||||
});
|
||||
|
||||
it('extracts EUR amounts with EUR prefix', () => {
|
||||
const entities = extractEntities('The minimum order is EUR 30.');
|
||||
const amountEntities = entities.filter(e => e.startsWith('amount:'));
|
||||
assert.ok(amountEntities.some(e => e === 'amount:30'), `Expected amount:30, got ${amountEntities}`);
|
||||
});
|
||||
|
||||
it('extracts EUR amounts with suffix', () => {
|
||||
const entities = extractEntities('Total cost: 150€');
|
||||
const amountEntities = entities.filter(e => e.startsWith('amount:'));
|
||||
assert.ok(amountEntities.some(e => e === 'amount:150'), `Expected amount:150, got ${amountEntities}`);
|
||||
});
|
||||
|
||||
it('extracts EUR amounts with "euros" suffix', () => {
|
||||
const entities = extractEntities('Shipping costs 5 euros.');
|
||||
const amountEntities = entities.filter(e => e.startsWith('amount:'));
|
||||
assert.ok(amountEntities.some(e => e === 'amount:5'), `Expected amount:5, got ${amountEntities}`);
|
||||
});
|
||||
|
||||
it('extracts percentages', () => {
|
||||
const entities = extractEntities('We offer a 10% discount for members.');
|
||||
const pctEntities = entities.filter(e => e.startsWith('pct:'));
|
||||
assert.ok(pctEntities.some(e => e === 'pct:10'), `Expected pct:10, got ${pctEntities}`);
|
||||
});
|
||||
|
||||
it('extracts time limits in days', () => {
|
||||
const entities = extractEntities('You have 30 days to return items.');
|
||||
const timeEntities = entities.filter(e => e.startsWith('time:'));
|
||||
assert.ok(timeEntities.length > 0, `Expected time entities, got ${entities}`);
|
||||
});
|
||||
|
||||
it('extracts time limits in months', () => {
|
||||
const entities = extractEntities('Warranty lasts 6 months.');
|
||||
const timeEntities = entities.filter(e => e.startsWith('time:'));
|
||||
assert.ok(timeEntities.length > 0, `Expected time entities, got ${entities}`);
|
||||
});
|
||||
|
||||
it('extracts EU country names', () => {
|
||||
const entities = extractEntities('We ship to all EU countries including Netherlands and Germany.');
|
||||
const countryEntities = entities.filter(e => e.startsWith('country:'));
|
||||
assert.ok(countryEntities.some(e => e === 'country:netherlands'), `Expected country:netherlands, got ${countryEntities}`);
|
||||
assert.ok(countryEntities.some(e => e === 'country:germany'), `Expected country:germany, got ${countryEntities}`);
|
||||
});
|
||||
|
||||
it('extracts English policy terms', () => {
|
||||
const entities = extractEntities('Our refund and return policy covers shipping.');
|
||||
const policyEntities = entities.filter(e => e.startsWith('policy:'));
|
||||
assert.ok(policyEntities.some(e => e === 'policy:refund'), `Expected policy:refund, got ${policyEntities}`);
|
||||
assert.ok(policyEntities.some(e => e === 'policy:return'), `Expected policy:return, got ${policyEntities}`);
|
||||
assert.ok(policyEntities.some(e => e === 'policy:shipping'), `Expected policy:shipping, got ${policyEntities}`);
|
||||
});
|
||||
|
||||
it('extracts number+unit combos', () => {
|
||||
const entities = extractEntities('Minimum order is 5 pieces.');
|
||||
const unitEntities = entities.filter(e => e.startsWith('unit:'));
|
||||
assert.ok(unitEntities.length > 0, `Expected unit entities, got ${entities}`);
|
||||
});
|
||||
|
||||
it('returns empty array for null/undefined text', () => {
|
||||
assert.deepStrictEqual(extractEntities(null), []);
|
||||
assert.deepStrictEqual(extractEntities(undefined), []);
|
||||
assert.deepStrictEqual(extractEntities(''), []);
|
||||
});
|
||||
|
||||
it('handles decimal amounts', () => {
|
||||
const entities = extractEntities('The fee is €4.95.');
|
||||
const amountEntities = entities.filter(e => e.startsWith('amount:'));
|
||||
assert.ok(amountEntities.some(e => e === 'amount:4.95'), `Expected amount:4.95, got ${amountEntities}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('computeCoverage', () => {
|
||||
it('finds covered entities in KB texts', () => {
|
||||
const replyEntities = ['amount:200', 'policy:shipping'];
|
||||
const kbHitTexts = ['Free shipping for orders over €200 within Netherlands.'];
|
||||
const { covered, uncovered } = computeCoverage(replyEntities, kbHitTexts);
|
||||
assert.ok(covered.includes('amount:200'), `Expected amount:200 in covered, got ${covered}`);
|
||||
assert.ok(covered.includes('policy:shipping'), `Expected policy:shipping in covered, got ${covered}`);
|
||||
assert.strictEqual(uncovered.length, 0);
|
||||
});
|
||||
|
||||
it('identifies uncovered entities', () => {
|
||||
const replyEntities = ['amount:500', 'policy:shipping', 'country:spain'];
|
||||
const kbHitTexts = ['Free shipping for orders over €200 within Netherlands.'];
|
||||
const { covered, uncovered } = computeCoverage(replyEntities, kbHitTexts);
|
||||
assert.ok(covered.includes('policy:shipping'), `Expected policy:shipping in covered`);
|
||||
assert.ok(uncovered.includes('amount:500'), `Expected amount:500 in uncovered`);
|
||||
assert.ok(uncovered.includes('country:spain'), `Expected country:spain in uncovered`);
|
||||
});
|
||||
|
||||
it('handles KB hit objects with question/answer', () => {
|
||||
const replyEntities = ['amount:200', 'policy:shipping'];
|
||||
const kbHitTexts = [{ question: 'Shipping policy?', answer: 'Free shipping for orders over €200.', category: 'shipping' }];
|
||||
const { covered } = computeCoverage(replyEntities, kbHitTexts);
|
||||
assert.ok(covered.includes('amount:200'));
|
||||
});
|
||||
|
||||
it('returns empty for empty reply entities', () => {
|
||||
const { covered, uncovered } = computeCoverage([], ['Some KB text']);
|
||||
assert.deepStrictEqual(covered, []);
|
||||
assert.deepStrictEqual(uncovered, []);
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkCitation', () => {
|
||||
it('returns null for non-KB intent (product_search)', () => {
|
||||
const result = checkCitation('Some product reply', ['KB text'], 'product_search');
|
||||
assert.strictEqual(result.citation_score, null);
|
||||
assert.strictEqual(result.hallucination_flag, null);
|
||||
});
|
||||
|
||||
it('returns null for non-KB intent (order_tracking)', () => {
|
||||
const result = checkCitation('Your order is shipped', [], 'order_tracking');
|
||||
assert.strictEqual(result.citation_score, null);
|
||||
assert.strictEqual(result.hallucination_flag, null);
|
||||
});
|
||||
|
||||
it('returns null for non-KB intent (inventory_check)', () => {
|
||||
const result = checkCitation('5 items in stock', [], 'inventory_check');
|
||||
assert.strictEqual(result.citation_score, null);
|
||||
assert.strictEqual(result.hallucination_flag, null);
|
||||
});
|
||||
|
||||
it('computes high citation_score for well-supported reply', () => {
|
||||
const reply = 'Free shipping for orders over €200 within the Netherlands. We offer a 30-day return policy.';
|
||||
const kbHits = [{ question: 'Shipping?', answer: 'Free shipping for orders over €200 within Netherlands.', category: 'shipping' }];
|
||||
const result = checkCitation(reply, kbHits, 'policy_faq');
|
||||
assert.ok(result.citation_score >= 0.5, `Expected score >= 0.5, got ${result.citation_score}`);
|
||||
assert.strictEqual(result.hallucination_flag, false);
|
||||
});
|
||||
|
||||
it('flags hallucination for low coverage reply', () => {
|
||||
const reply = 'Free shipping for orders over €200 to ALL EU countries including Spain and Italy. 50% discount on membership.';
|
||||
const kbHits = ['Free shipping for orders over €200 within Netherlands only. No special discounts.'];
|
||||
const result = checkCitation(reply, kbHits, 'policy_faq');
|
||||
assert.strictEqual(result.hallucination_flag, true);
|
||||
assert.ok(result.citation_score < 0.5, `Expected score < 0.5, got ${result.citation_score}`);
|
||||
assert.strictEqual(result.severity, 'low_coverage');
|
||||
});
|
||||
|
||||
it('returns citation_score=0 and hallucination_flag=true for no KB hits with policy entities', () => {
|
||||
const reply = 'Our refund policy allows returns within 30 days. Shipping costs €5.99.';
|
||||
const result = checkCitation(reply, [], 'policy_faq');
|
||||
assert.strictEqual(result.citation_score, 0);
|
||||
assert.strictEqual(result.hallucination_flag, true);
|
||||
assert.strictEqual(result.severity, 'no_kb_support');
|
||||
});
|
||||
|
||||
it('returns null for no KB hits and no factual entities', () => {
|
||||
const reply = 'Hello! How can I help you today?';
|
||||
const result = checkCitation(reply, [], 'general');
|
||||
assert.strictEqual(result.citation_score, null);
|
||||
assert.strictEqual(result.hallucination_flag, null);
|
||||
});
|
||||
|
||||
it('returns null for null/empty reply text', () => {
|
||||
const result = checkCitation(null, ['KB text'], 'policy_faq');
|
||||
assert.strictEqual(result.citation_score, null);
|
||||
assert.strictEqual(result.hallucination_flag, null);
|
||||
});
|
||||
|
||||
it('returns citation_score=1 when no entities are extracted from reply', () => {
|
||||
const reply = 'Yes, that is correct.';
|
||||
const kbHits = ['Some KB answer about something.'];
|
||||
const result = checkCitation(reply, kbHits, 'policy_faq');
|
||||
// No entities = nothing to hallucinate about
|
||||
assert.strictEqual(result.citation_score, 1);
|
||||
assert.strictEqual(result.hallucination_flag, false);
|
||||
});
|
||||
|
||||
it('handles general intent with KB hits', () => {
|
||||
const reply = 'You can get a refund within 30 days.';
|
||||
const kbHits = ['Our refund policy: 30-day return window.'];
|
||||
const result = checkCitation(reply, kbHits, 'general');
|
||||
assert.ok(result.citation_score >= 0.5);
|
||||
});
|
||||
});
|
||||
|
||||
describe('detectFabricatedCitations', () => {
|
||||
it('detects cross-domain URLs not in KB sources', () => {
|
||||
const reply = 'See our policy at https://evil-competitor.com/fake-policy for details.';
|
||||
const kbSources = ['https://yehwang.com/real-policy'];
|
||||
const fabricated = detectFabricatedCitations(reply, kbSources);
|
||||
assert.strictEqual(fabricated.length, 1);
|
||||
assert.ok(fabricated[0].includes('evil-competitor.com'));
|
||||
});
|
||||
|
||||
it('allows same-origin navigation links not in KB sources (站内导航不算编造)', () => {
|
||||
const reply = 'Browse items on https://dropshipping.yehwang.com';
|
||||
const kbSources = ['https://dropshipping.yehwang.com/faq/returns'];
|
||||
const fabricated = detectFabricatedCitations(reply, kbSources);
|
||||
assert.strictEqual(fabricated.length, 0);
|
||||
});
|
||||
|
||||
it('returns empty when all reply URLs are in KB sources', () => {
|
||||
const reply = 'See https://yehwang.com/policy for details.';
|
||||
const kbSources = ['https://yehwang.com/policy'];
|
||||
const fabricated = detectFabricatedCitations(reply, kbSources);
|
||||
assert.strictEqual(fabricated.length, 0);
|
||||
});
|
||||
|
||||
it('returns empty for reply with no URLs', () => {
|
||||
const reply = 'Free shipping for orders over €200.';
|
||||
const fabricated = detectFabricatedCitations(reply, ['https://yehwang.com/policy']);
|
||||
assert.strictEqual(fabricated.length, 0);
|
||||
});
|
||||
|
||||
it('returns empty for null/undefined inputs', () => {
|
||||
assert.deepStrictEqual(detectFabricatedCitations(null, []), []);
|
||||
assert.deepStrictEqual(detectFabricatedCitations('', null), []);
|
||||
assert.deepStrictEqual(detectFabricatedCitations('', []), []);
|
||||
});
|
||||
|
||||
it('flags fabricated cross-domain citation as severe hallucination in checkCitation', () => {
|
||||
const reply = 'See our policy at https://other-site.com/fake-page for details about €200 free shipping.';
|
||||
const kbHits = ['Free shipping over €200.'];
|
||||
const kbSources = ['https://yehwang.com/real-shipping-policy'];
|
||||
const result = checkCitation(reply, kbHits, 'policy_faq', kbSources);
|
||||
assert.strictEqual(result.hallucination_flag, true);
|
||||
assert.strictEqual(result.severity, 'fabricated_citation');
|
||||
assert.strictEqual(result.citation_score, 0);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,24 @@
|
||||
'use strict';
|
||||
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { createEscalation } = require('../src/escalation');
|
||||
|
||||
const esc = createEscalation({ client: { sendPrivateNote: async () => {}, assignUser: async () => {} } });
|
||||
|
||||
test('escalation: three consecutive fails -> level 2', () => {
|
||||
assert.equal(esc.shouldEscalate({ messagesProcessed: 3 }, 'policy_faq', 'high', 3, 2).level, 2);
|
||||
});
|
||||
|
||||
test('escalation: low KB with no hits -> level 1', () => {
|
||||
assert.equal(esc.shouldEscalate({ messagesProcessed: 1 }, 'policy_faq', 'low', 0, 0).level, 1);
|
||||
});
|
||||
|
||||
test('escalation: healthy -> level 0', () => {
|
||||
assert.equal(esc.shouldEscalate({ messagesProcessed: 1 }, 'policy_faq', 'high', 0, 3).level, 0);
|
||||
});
|
||||
|
||||
test('escalation: appends localized human option', () => {
|
||||
const out = esc.appendEscalationOption('您好', '您好，订单怎么查');
|
||||
assert.match(out, /转人工/);
|
||||
});
|
||||
@@ -0,0 +1,10 @@
|
||||
'use strict';
|
||||
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { createExperimentManager } = require('../src/experiments');
|
||||
|
||||
test('experiments: no config returns no variant', () => {
|
||||
const m = createExperimentManager({ experimentsPath: '/nonexistent.json' });
|
||||
assert.equal(m.getVariant('c1').variant, null);
|
||||
});
|
||||
@@ -0,0 +1,11 @@
|
||||
'use strict';
|
||||
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { createFaqRefiner } = require('../src/faq-refiner');
|
||||
|
||||
test('faq refiner: returns only Q blocks', async () => {
|
||||
const llm = { async chat() { return { content: '### Q: What?\nAnswer.' }; } };
|
||||
const out = await createFaqRefiner(llm).refineToFaq('page content', 'test');
|
||||
assert.match(out, /^### Q:/);
|
||||
});
|
||||
@@ -0,0 +1,15 @@
|
||||
'use strict';
|
||||
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { detectLanguage, languageMandate } = require('../src/language');
|
||||
|
||||
test('language: detects common languages', () => {
|
||||
assert.equal(detectLanguage('你好，订单怎么查'), 'Chinese');
|
||||
assert.equal(detectLanguage('Wie ist der Preis?'), 'German');
|
||||
assert.equal(detectLanguage('What is the minimum order?'), 'English');
|
||||
});
|
||||
|
||||
test('language: mandate names latest language', () => {
|
||||
assert.match(languageMandate('你好'), /Chinese/);
|
||||
});
|
||||
@@ -0,0 +1,18 @@
|
||||
'use strict';
|
||||
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { createReranker } = require('../src/reranker');
|
||||
|
||||
test('reranker: returns LLM-selected candidates', async () => {
|
||||
const llm = { async chat() { return { content: '2,0' }; } };
|
||||
const candidates = [{ question: 'a' }, { question: 'b' }, { question: 'c' }];
|
||||
const out = await createReranker(llm).rerank('q', candidates, 2);
|
||||
assert.deepEqual(out.map((x) => x.question), ['c', 'a']);
|
||||
});
|
||||
|
||||
test('reranker: short list passes through', async () => {
|
||||
const candidates = [{ question: 'a' }];
|
||||
const out = await createReranker({ chat: async () => ({ content: '' }) }).rerank('q', candidates, 2);
|
||||
assert.deepEqual(out, candidates);
|
||||
});
|
||||
@@ -0,0 +1,11 @@
|
||||
'use strict';
|
||||
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { otpEmail } = require('../src/verification');
|
||||
|
||||
test('otp email is localized', () => {
|
||||
assert.match(otpEmail('zh', '123456').text, /验证码/);
|
||||
assert.match(otpEmail('de', '123456').text, /Bestätigungscode/);
|
||||
assert.match(otpEmail('en', '123456').text, /verification code/);
|
||||
});
|
||||
Reference in New Issue
Block a user