feat: port quality, escalation, KB version, site-map KB, experiments and metrics
This commit is contained in:
wangliang
+120
-3
@@ -12,10 +12,17 @@ const { markdownToHtml } = require('./markdown');
|
||||
const { guardReply } = require('./output-guard');
|
||||
const { quickClassify, detectEmotion, INTENTS } = require('./intent');
|
||||
const { createContextManager } = require('./context');
|
||||
const { createReranker } = require('./reranker');
|
||||
const { detectInjection, markToolResult, INJECTION_CAUTION_PROMPT } = require('./security');
|
||||
const { hasBuyingIntent, missingLeadContacts, detectLanguage, appendLeadFollowup, alreadyAsksForLead } = require('./lead-intent');
|
||||
const { createLlmCache } = require('./llm-cache');
|
||||
const { checkCitation } = require('./citation-checker');
|
||||
const { createFlywheelWriter } = require('./flywheel');
|
||||
const { languageMandate } = require('./language');
|
||||
const { createEscalation } = require('./escalation');
|
||||
const { createExperimentManager } = require('./experiments');
|
||||
const crypto = require('crypto');
|
||||
const prom = require('./prom');
|
||||
|
||||
const MAX_TOOL_STEPS = 4;
|
||||
|
||||
@@ -40,7 +47,18 @@ function createAgent(cfg, deps = {}) {
|
||||
const metrics = deps.metrics || createMetrics(cfg);
|
||||
const contextManager = deps.contextManager || createContextManager();
|
||||
const llmCache = deps.llmCache || createLlmCache(cfg);
|
||||
const tools = deps.tools || createTools(cfg, { verification, retriever, client });
|
||||
const escalation = deps.escalation || createEscalation({ client, handoffUserId: cfg.handoffUserId || 1 });
|
||||
const experiments = deps.experiments || createExperimentManager(cfg);
|
||||
const flywheel = deps.flywheel || createFlywheelWriter({
|
||||
pool: metrics.pool,
|
||||
callLLM: async (messages, system, maxTokens) => {
|
||||
const msgs = system ? [{ role: 'system', content: system }, ...messages] : messages;
|
||||
const res = await llm.chat({ messages: msgs, maxTokens: maxTokens || 300 });
|
||||
return { text: res.content || '' };
|
||||
},
|
||||
});
|
||||
const reranker = deps.reranker || createReranker(llm);
|
||||
const tools = deps.tools || createTools(cfg, { verification, retriever, client, reranker });
|
||||
const leadStore = deps.leadStore || (tools && tools.leadStore) || null;
|
||||
const byName = new Map(tools.map((t) => [t.name, t]));
|
||||
const toolDefs = tools.map((t) => ({
|
||||
@@ -87,6 +105,7 @@ function createAgent(cfg, deps = {}) {
|
||||
const startedAt = Date.now();
|
||||
const stats = { llmCalls: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0, toolCalls: [], toolResults: [] };
|
||||
const profile = resolveSiteProfile(cfg, { inboxId: String(conv.inbox_id || '') });
|
||||
const variant = experiments.getVariant(conv.uuid);
|
||||
const identity = await identityFor(conv);
|
||||
const ctx = {
|
||||
verified: identity.verified,
|
||||
@@ -97,6 +116,7 @@ function createAgent(cfg, deps = {}) {
|
||||
};
|
||||
|
||||
const currentText = (webhookMessage.text_content || stripHtml(webhookMessage.content)).trim();
|
||||
ctx.language = detectLanguage(currentText);
|
||||
const messages = await buildMessages(conv, webhookMessage);
|
||||
if (!messages.length) return null;
|
||||
const injection = detectInjection(currentText);
|
||||
@@ -107,6 +127,8 @@ function createAgent(cfg, deps = {}) {
|
||||
channel: conv.channel || 'livechat',
|
||||
}) + (injection.level === 'high' ? INJECTION_CAUTION_PROMPT : '');
|
||||
|
||||
systemContent += languageMandate(currentText);
|
||||
|
||||
// 购买意向：LLM 之前先给硬指令，LLM 之后还有程序化兜底
|
||||
let leadState = null;
|
||||
if (leadStore) {
|
||||
@@ -126,6 +148,8 @@ function createAgent(cfg, deps = {}) {
|
||||
// 预检索：政策问题先查 KB，商品问题先查商品，避免模型跳过工具直接编。
|
||||
const intent = quickClassify(currentText);
|
||||
const emotion = detectEmotion(currentText);
|
||||
const emotionStrategy = escalation.applyEmotionStrategy(emotion);
|
||||
const maxSteps = emotionStrategy.maxToolTurns || MAX_TOOL_STEPS;
|
||||
if (emotion !== 'neutral') {
|
||||
const toneLine = emotion === 'angry'
|
||||
? 'TONE: The customer is angry. Acknowledge their frustration, be empathetic, and escalate/hand off if needed.'
|
||||
@@ -180,12 +204,19 @@ function createAgent(cfg, deps = {}) {
|
||||
toolCalls: stats.toolCalls, audit: { ...auditAnswer({ finalContent: cached, toolResults: stats.toolResults, toolCalls: stats.toolCalls }), cache_hit: true },
|
||||
replyLength: cached.length,
|
||||
});
|
||||
prom.inc('bridge_ai_cache_hits_total');
|
||||
console.log(JSON.stringify({ ts: new Date().toISOString(), agent: 'cache_hit', conversation: conv.uuid, length: cached.length }));
|
||||
return cached;
|
||||
}
|
||||
}
|
||||
for (let step = 0; step < MAX_TOOL_STEPS; step += 1) {
|
||||
const response = await llm.chat({ messages, tools: toolDefs, temperature: 0.3, maxTokens: 1200 });
|
||||
for (let step = 0; step < maxSteps; step += 1) {
|
||||
prom.inc('bridge_ai_llm_calls_total');
|
||||
const response = await llm.chat({
|
||||
messages,
|
||||
tools: toolDefs,
|
||||
temperature: (variant.overriddenParams && variant.overriddenParams.temperature) ?? 0.3,
|
||||
maxTokens: (variant.overriddenParams && variant.overriddenParams.maxTokens) ?? 1200,
|
||||
});
|
||||
stats.llmCalls += 1;
|
||||
if (response.usage) {
|
||||
stats.promptTokens += Number(response.usage.prompt_tokens) || 0;
|
||||
@@ -210,6 +241,7 @@ function createAgent(cfg, deps = {}) {
|
||||
} catch (err) {
|
||||
result = { error: err.code || err.message || 'tool_failed', message: err.message || String(err) };
|
||||
}
|
||||
prom.inc('bridge_ai_tool_calls_total', { tool: name, ok: !(result && result.error) });
|
||||
stats.toolCalls.push({ name, ms: Date.now() - started, ok: !(result && result.error) });
|
||||
stats.toolResults.push(result);
|
||||
console.log(JSON.stringify({
|
||||
@@ -256,6 +288,7 @@ function createAgent(cfg, deps = {}) {
|
||||
toolCalls: stats.toolCalls,
|
||||
lang: detectLanguage(currentText),
|
||||
});
|
||||
if (guard.blocked) prom.inc('bridge_ai_guard_blocked_total', { reason: (guard.reasons || []).join('|') || 'unknown' });
|
||||
if (guard.blocked || guard.removed_urls.length) {
|
||||
console.log(JSON.stringify({
|
||||
ts: new Date().toISOString(),
|
||||
@@ -270,6 +303,57 @@ function createAgent(cfg, deps = {}) {
|
||||
}
|
||||
finalContent = guard.content;
|
||||
|
||||
// 引用一致性检查（语义/数字覆盖）+ 飞轮
|
||||
const kbHits = [];
|
||||
const kbSources = [];
|
||||
(stats.toolCalls || []).forEach((call, i) => {
|
||||
if (call.name !== 'search_kb') return;
|
||||
const result = stats.toolResults[i];
|
||||
for (const hit of (result && result.results) || []) {
|
||||
kbHits.push(hit);
|
||||
for (const u of hit.source_urls || []) kbSources.push(u);
|
||||
}
|
||||
});
|
||||
const citation = checkCitation(finalContent, kbHits, intent, [...new Set(kbSources)]);
|
||||
|
||||
// 三级升级 + 会话状态
|
||||
let escalationLevel = 0;
|
||||
let escalationReason = null;
|
||||
try {
|
||||
const latestState = (leadStore && await leadStore.get(conv.uuid)) || {};
|
||||
const state = {
|
||||
...latestState,
|
||||
messagesProcessed: (Number(latestState.messagesProcessed) || 0) + 1,
|
||||
consecutiveFails: Number(latestState.consecutiveFails) || 0,
|
||||
};
|
||||
const topScore = kbHits.length ? Number(kbHits[0].score) || 0 : 0;
|
||||
const kbConfidence = topScore >= 0.7 ? 'high' : topScore >= 0.55 ? 'medium' : topScore > 0 ? 'low' : 'none';
|
||||
const decision = escalation.shouldEscalate(
|
||||
latestState,
|
||||
intent,
|
||||
kbConfidence,
|
||||
Number(latestState.consecutiveFails) || 0,
|
||||
kbHits.length
|
||||
);
|
||||
escalationLevel = decision.level;
|
||||
prom.inc('bridge_ai_escalations_total', { level: decision.level });
|
||||
escalationReason = decision.reason;
|
||||
if (decision.level === 1 && !/handoff|human|人工/i.test(finalContent)) {
|
||||
finalContent = escalation.appendEscalationOption(finalContent, currentText);
|
||||
} else if (decision.level === 2) {
|
||||
await escalation.escalateToHuman({ convUuid: conv.uuid, trigger: decision.reason, history: messages.slice(-6) });
|
||||
const zh = /[\u4e00-\u9fff]/.test(currentText);
|
||||
const line = zh ? '\n\n已为您转接人工同事，请稍候。' : '\n\nI have transferred you to a human colleague.';
|
||||
if (!finalContent.includes(line.trim())) finalContent += line;
|
||||
}
|
||||
state.consecutiveFails = (kbConfidence === 'low' || kbConfidence === 'none' || citation.hallucination_flag === true) ? state.consecutiveFails + 1 : 0;
|
||||
state.kbConfidence = kbConfidence;
|
||||
if (leadStore && leadStore.saveState) await leadStore.saveState(conv.uuid, state);
|
||||
else if (leadStore && leadStore.save) await leadStore.save(conv.uuid, state.slots || {});
|
||||
} catch (err) {
|
||||
console.error(JSON.stringify({ ts: new Date().toISOString(), warn: `escalation failed: ${err.message}` }));
|
||||
}
|
||||
|
||||
const alreadyHtml = /<(p|br|a|ul|ol|strong|div)\b/i.test(finalContent);
|
||||
const outboundContent = alreadyHtml ? finalContent : markdownToHtml(finalContent);
|
||||
await client.sendMessage(conv.uuid, outboundContent, { echoId: `bridge-ai-${webhookMessage.uuid}` });
|
||||
@@ -284,8 +368,40 @@ function createAgent(cfg, deps = {}) {
|
||||
reply: finalContent,
|
||||
});
|
||||
}
|
||||
if (citation.hallucination_flag === true || (citation.citation_score !== null && citation.citation_score < 0.5)) {
|
||||
try {
|
||||
const sampleId = await flywheel.captureSample({
|
||||
convId: conv.uuid,
|
||||
query: currentText,
|
||||
reply: finalContent,
|
||||
intent,
|
||||
kbHits: kbHits.slice(0, 10),
|
||||
signal: citation.hallucination_flag === true ? 'hallucination' : 'low_citation',
|
||||
citationScore: citation.citation_score,
|
||||
});
|
||||
if (sampleId) {
|
||||
setImmediate(async () => {
|
||||
try {
|
||||
const review = await flywheel.preLabel({ query: currentText, reply: finalContent, kbHits });
|
||||
if (review) await flywheel.updateReview(sampleId, review);
|
||||
} catch { /* noop */ }
|
||||
});
|
||||
}
|
||||
await client.sendPrivateNote(conv.uuid, `[⚠️ Citation Alert] score=${citation.citation_score}, severity=${citation.severity || 'low'}. This reply may need review.`).catch(() => {});
|
||||
console.log(JSON.stringify({ ts: new Date().toISOString(), agent: 'citation_alert', conversation: conv.uuid, citation_score: citation.citation_score, severity: citation.severity }));
|
||||
} catch (err) {
|
||||
console.error(JSON.stringify({ ts: new Date().toISOString(), warn: `citation/flywheel failed: ${err.message}` }));
|
||||
}
|
||||
}
|
||||
|
||||
const audit = auditAnswer({ finalContent, toolResults: stats.toolResults, toolCalls: stats.toolCalls });
|
||||
audit.injection_level = injection.level;
|
||||
audit.escalation_level = escalationLevel;
|
||||
audit.escalation_reason = escalationReason;
|
||||
audit.experiment = { id: variant.experiment_id, variant: variant.variant };
|
||||
audit.citation_score = citation.citation_score;
|
||||
audit.hallucination_flag = citation.hallucination_flag;
|
||||
audit.uncovered_entities = citation.uncovered_entities;
|
||||
audit.lead_followup = leadFollowup;
|
||||
audit.output_guard = { blocked: guard.blocked, rewritten: guard.rewritten, reasons: guard.reasons, removed_urls: guard.removed_urls, evidence_missing: guard.evidence_missing };
|
||||
await metrics.record({
|
||||
@@ -303,6 +419,7 @@ function createAgent(cfg, deps = {}) {
|
||||
audit,
|
||||
replyLength: finalContent.length,
|
||||
});
|
||||
prom.inc('bridge_ai_runs_total');
|
||||
console.log(JSON.stringify({ ts: new Date().toISOString(), agent: 'replied', conversation: conv.uuid, length: finalContent.length, audit }));
|
||||
return finalContent;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user