feat: port quality, escalation, KB version, site-map KB, experiments and metrics
This commit is contained in:
wangliang
@@ -0,0 +1,314 @@
|
||||
// citation_checker.js — Entity extraction + coverage calculation + hallucination flagging
|
||||
// Shared between offline eval and online scoring paths
|
||||
|
||||
const EU_COUNTRIES = new Set([
|
||||
'austria', 'belgium', 'bulgaria', 'croatia', 'cyprus', 'czech republic', 'czechia',
|
||||
'denmark', 'estonia', 'finland', 'france', 'germany', 'greece', 'hungary', 'ireland',
|
||||
'italy', 'latvia', 'lithuania', 'luxembourg', 'malta', 'netherlands', 'poland',
|
||||
'portugal', 'romania', 'slovakia', 'slovenia', 'spain', 'sweden',
|
||||
// Also common forms / abbreviations
|
||||
'uk', 'united kingdom', 'england', 'scotland', 'wales', 'northern ireland',
|
||||
// German names
|
||||
'deutschland', 'österreich', 'schweiz',
|
||||
// French names
|
||||
'allemagne', 'pays-bas', 'espagne', 'italie', 'belgique',
|
||||
// Dutch names
|
||||
'duitsland', 'nederland', 'belgië', 'spanje', 'italië', 'frankrijk',
|
||||
]);
|
||||
|
||||
const POLICY_TERMS_EN = [
|
||||
'refund', 'return', 'shipping', 'discount', 'vat', 'membership', 'dropship',
|
||||
'factory-direct', 'pre-order', 'preorder', 'backorder', 'cancellation', 'cancel',
|
||||
'exchange', 'warranty', 'guarantee', 'minimum order', 'bulk order', 'wholesale',
|
||||
'retail', 'reseller', 'business', 'corporate', 'invoice', 'payment', 'delivery',
|
||||
'pickup', 'lead time', 'turnaround', 'restock', 'reorder', 'custom', 'personalize',
|
||||
'engrave', 'sample', 'catalog', 'brochure', 'newsletter', 'subscribe', 'unsubscribe',
|
||||
'account', 'register', 'login', 'sign up', 'password',
|
||||
];
|
||||
|
||||
// Multilingual policy term translations (keyed by normalized term)
|
||||
const POLICY_TERMS_I18N = {
|
||||
// German
|
||||
'rückerstattung': 'refund', 'rückgabe': 'return', 'versand': 'shipping',
|
||||
'versandkosten': 'shipping', 'rabatt': 'discount', 'mehrwertsteuer': 'vat',
|
||||
'mitgliedschaft': 'membership', 'stornierung': 'cancellation', 'stornieren': 'cancel',
|
||||
'rechnung': 'invoice', 'zahlung': 'payment', 'lieferung': 'delivery',
|
||||
'lieferzeit': 'delivery', 'großhandel': 'wholesale', 'katalog': 'catalog',
|
||||
'registrieren': 'register', 'konto': 'account',
|
||||
// French
|
||||
'remboursement': 'refund', 'retour': 'return', 'expédition': 'shipping',
|
||||
'frais de livraison': 'shipping', 'livraison': 'delivery', 'remise': 'discount',
|
||||
'réduction': 'discount', 'tva': 'vat', 'adhésion': 'membership',
|
||||
'annulation': 'cancellation', 'annuler': 'cancel', 'facture': 'invoice',
|
||||
'paiement': 'payment', 'grossiste': 'wholesale', 'catalogue': 'catalog',
|
||||
'inscription': 'register', 'compte': 'account',
|
||||
// Dutch
|
||||
'terugbetaling': 'refund', 'retour': 'return', 'retourneren': 'return',
|
||||
'verzending': 'shipping', 'verzendkosten': 'shipping', 'korting': 'discount',
|
||||
'btw': 'vat', 'lidmaatschap': 'membership', 'annulering': 'cancellation',
|
||||
'annuleren': 'cancel', 'factuur': 'invoice', 'betaling': 'payment',
|
||||
'groothandel': 'wholesale', 'catalogus': 'catalog', 'registreren': 'register',
|
||||
'account': 'account',
|
||||
// Italian
|
||||
'rimborso': 'refund', 'reso': 'return', 'spedizione': 'shipping',
|
||||
'costi di spedizione': 'shipping', 'sconto': 'discount', 'iva': 'vat',
|
||||
'iscrizione': 'membership', 'cancellazione': 'cancellation', 'cancellare': 'cancel',
|
||||
'fattura': 'invoice', 'pagamento': 'payment', 'ingrosso': 'wholesale',
|
||||
'catalogo': 'catalog', 'registrare': 'register',
|
||||
};
|
||||
|
||||
// Non-KB intents that should skip hallucination checking
|
||||
const NON_KB_INTENTS = new Set(['product_search', 'order_tracking', 'inventory_check', 'sales_lead']);
|
||||
|
||||
/**
|
||||
* Extract key entities from text.
|
||||
* Entities: amounts (€/EUR), percentages, time limits (days/weeks/months),
|
||||
* country names, policy terms, number+unit combos.
|
||||
*/
|
||||
function extractEntities(text) {
|
||||
if (!text || typeof text !== 'string') return [];
|
||||
const lower = text.toLowerCase();
|
||||
const entities = new Set();
|
||||
|
||||
// Amounts: €200, EUR 200, 200€, 200 euros, 200 eur
|
||||
const amountMatches = text.matchAll(/(?:€|EUR\s?|euros?\s?)\s?(\d+(?:[.,]\d+)?)/gi);
|
||||
for (const m of amountMatches) {
|
||||
entities.add(`amount:${m[1].replace(',', '.')}`);
|
||||
}
|
||||
const amountMatches2 = text.matchAll(/(\d+(?:[.,]\d+)?)\s?(?:€|EUR|euros?|eur)/gi);
|
||||
for (const m of amountMatches2) {
|
||||
entities.add(`amount:${m[1].replace(',', '.')}`);
|
||||
}
|
||||
|
||||
// Percentages: 10%, 10 percent
|
||||
const pctMatches = text.matchAll(/(\d+(?:[.,]\d+)?)\s*(?:%|percent)/gi);
|
||||
for (const m of pctMatches) {
|
||||
entities.add(`pct:${m[1].replace(',', '.')}`);
|
||||
}
|
||||
|
||||
// Time limits: 30 days, 2 weeks, 3 months, 14 Tage, 30 jours, 30 dagen, 5-7 werkdagen
|
||||
const timeMatches = text.matchAll(/(\d+(?:\s*[-–]\s*\d+)?)\s*(?:days?|weeks?|months?|werktage?|werkdagen?|jours?|giorni?|dagen?|Tage?|settimane?|mois)/gi);
|
||||
for (const m of timeMatches) {
|
||||
entities.add(`time:${m[0].toLowerCase().replace(/\s+/g, '')}`);
|
||||
}
|
||||
|
||||
// Number + unit combos: 5kg, 3 pieces, 10 items, 2 kg, 5 stück, 3 pièces, 4 stuks
|
||||
const unitMatches = text.matchAll(/(\d+(?:[.,]\d+)?)\s*(?:kg|g|lb|oz|pieces?|items?|units?|stück|pièces?|stuks?|pairs?|sets?|pcs)/gi);
|
||||
for (const m of unitMatches) {
|
||||
entities.add(`unit:${m[0].toLowerCase().replace(/\s+/g, '')}`);
|
||||
}
|
||||
|
||||
// EU country names
|
||||
const words = lower.split(/[\s,.;:!?"'()\-/]+/).filter(Boolean);
|
||||
// Also check bigrams for "united kingdom", "czech republic"
|
||||
for (let i = 0; i < words.length; i++) {
|
||||
if (EU_COUNTRIES.has(words[i])) {
|
||||
entities.add(`country:${words[i]}`);
|
||||
}
|
||||
if (i + 1 < words.length) {
|
||||
const bigram = words[i] + ' ' + words[i + 1];
|
||||
if (EU_COUNTRIES.has(bigram)) {
|
||||
entities.add(`country:${bigram}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// English policy terms (whole word match)
|
||||
for (const term of POLICY_TERMS_EN) {
|
||||
const pattern = new RegExp('\\b' + term.replace(/[- ]/g, '[- ]') + '\\b', 'i');
|
||||
if (pattern.test(text)) {
|
||||
entities.add(`policy:${term.toLowerCase()}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Multilingual policy terms
|
||||
for (const [foreignTerm, englishTerm] of Object.entries(POLICY_TERMS_I18N)) {
|
||||
if (lower.includes(foreignTerm.toLowerCase())) {
|
||||
entities.add(`policy:${englishTerm}`);
|
||||
}
|
||||
}
|
||||
|
||||
return [...entities];
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize entities for cross-language comparison.
|
||||
* Maps multilingual entities to their English equivalents.
|
||||
*/
|
||||
function normalizeEntity(entity) {
|
||||
// Already prefixed with type: e.g., "policy:refund"
|
||||
return entity;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if entities from the reply are covered by KB hit texts.
|
||||
* Returns { covered, uncovered } entity arrays.
|
||||
*/
|
||||
function computeCoverage(replyEntities, kbHitTexts) {
|
||||
if (!replyEntities.length) return { covered: [], uncovered: [] };
|
||||
|
||||
// Collect all entities from KB texts
|
||||
const kbEntities = new Set();
|
||||
for (const hit of kbHitTexts) {
|
||||
const text = typeof hit === 'string' ? hit : [hit.question, hit.answer, hit.category].filter(Boolean).join(' ');
|
||||
const extracted = extractEntities(text);
|
||||
for (const e of extracted) {
|
||||
kbEntities.add(e);
|
||||
}
|
||||
}
|
||||
|
||||
const covered = [];
|
||||
const uncovered = [];
|
||||
for (const entity of replyEntities) {
|
||||
if (kbEntities.has(entity)) {
|
||||
covered.push(entity);
|
||||
} else {
|
||||
uncovered.push(entity);
|
||||
}
|
||||
}
|
||||
|
||||
return { covered, uncovered };
|
||||
}
|
||||
|
||||
/**
|
||||
* Check for fabricated citations — URLs in the reply that are NOT in the KB sources.
|
||||
* Returns array of fabricated URLs found.
|
||||
*/
|
||||
function detectFabricatedCitations(replyText, kbSources) {
|
||||
if (!replyText || !kbSources || !kbSources.length) return [];
|
||||
|
||||
// Extract URLs from reply
|
||||
const urlMatches = replyText.matchAll(/https?:\/\/[^\s)\]>"']+/gi);
|
||||
const replyUrls = new Set();
|
||||
for (const m of urlMatches) {
|
||||
replyUrls.add(m[0].toLowerCase().replace(/\/+$/, ''));
|
||||
}
|
||||
|
||||
// Normalize KB source URLs
|
||||
const kbUrls = new Set();
|
||||
for (const url of kbSources) {
|
||||
if (typeof url === 'string') {
|
||||
kbUrls.add(url.toLowerCase().replace(/\/+$/, ''));
|
||||
}
|
||||
}
|
||||
|
||||
// Fabricated = URLs in reply not found in KB sources.
|
||||
// 允许"与 KB 来源同源(根域名)"的链接——AI 附加的站内导航链接(如 dropshipping.yehwang.com)
|
||||
// 不算编造引用；只有引用了一个完全不同的域名/来源才标记。
|
||||
const kbOrigins = new Set();
|
||||
for (const url of kbUrls) {
|
||||
try {
|
||||
kbOrigins.add(new URL(url).origin.toLowerCase());
|
||||
} catch (e) { /* 忽略非法 URL */ }
|
||||
}
|
||||
const fabricated = [];
|
||||
for (const url of replyUrls) {
|
||||
if (kbUrls.has(url)) continue;
|
||||
try {
|
||||
const origin = new URL(url).origin.toLowerCase();
|
||||
if (kbOrigins.has(origin)) continue; // 同源站内链接 → 不算编造
|
||||
} catch (e) { /* 非法 URL 跳过 */ }
|
||||
fabricated.push(url);
|
||||
}
|
||||
|
||||
return fabricated;
|
||||
}
|
||||
|
||||
/**
|
||||
* Main entry point: check citation consistency of a reply against KB hits.
|
||||
*
|
||||
* @param {string} replyText - Agent's final reply text (stripped of HTML)
|
||||
* @param {Array<string|object>} kbHitTexts - KB hit texts (strings or objects with question/answer/category)
|
||||
* @param {string} intent - Current intent label
|
||||
* @param {Array<string>} [kbSources] - KB source URLs (for fabricated citation detection)
|
||||
* @returns {{ citation_score: number|null, hallucination_flag: boolean|null, extracted_entities: string[], covered_entities: string[], uncovered_entities: string[], severity: string|null }}
|
||||
*/
|
||||
function checkCitation(replyText, kbHitTexts, intent, kbSources) {
|
||||
// Non-KB intents: skip hallucination checking
|
||||
if (NON_KB_INTENTS.has(intent)) {
|
||||
return {
|
||||
citation_score: null,
|
||||
hallucination_flag: null,
|
||||
extracted_entities: [],
|
||||
covered_entities: [],
|
||||
uncovered_entities: [],
|
||||
severity: null,
|
||||
};
|
||||
}
|
||||
|
||||
// No reply text: nothing to check
|
||||
if (!replyText || typeof replyText !== 'string') {
|
||||
return {
|
||||
citation_score: null,
|
||||
hallucination_flag: null,
|
||||
extracted_entities: [],
|
||||
covered_entities: [],
|
||||
uncovered_entities: [],
|
||||
severity: null,
|
||||
};
|
||||
}
|
||||
|
||||
// Check for fabricated citations first (most severe)
|
||||
const fabricatedUrls = detectFabricatedCitations(replyText, kbSources || []);
|
||||
if (fabricatedUrls.length > 0) {
|
||||
return {
|
||||
citation_score: 0,
|
||||
hallucination_flag: true,
|
||||
extracted_entities: [],
|
||||
covered_entities: [],
|
||||
uncovered_entities: [],
|
||||
severity: 'fabricated_citation',
|
||||
};
|
||||
}
|
||||
|
||||
// No KB hits but KB-intent (policy_faq/general): if reply has substantive entities, flag
|
||||
const hasKbHits = kbHitTexts && kbHitTexts.length > 0;
|
||||
const extractedEntities = extractEntities(replyText);
|
||||
|
||||
if (!hasKbHits) {
|
||||
// No KB data to check against — if there are policy entities, likely hallucinated
|
||||
const hasPolicyEntities = extractedEntities.some(e => e.startsWith('policy:'));
|
||||
const hasFactualEntities = extractedEntities.some(e => e.startsWith('amount:') || e.startsWith('pct:') || e.startsWith('time:'));
|
||||
|
||||
if (hasPolicyEntities || hasFactualEntities) {
|
||||
return {
|
||||
citation_score: 0,
|
||||
hallucination_flag: true,
|
||||
extracted_entities: extractedEntities,
|
||||
covered_entities: [],
|
||||
uncovered_entities: extractedEntities,
|
||||
severity: 'no_kb_support',
|
||||
};
|
||||
}
|
||||
|
||||
// No entities extracted, no KB hits — can't determine, not flagged
|
||||
return {
|
||||
citation_score: null,
|
||||
hallucination_flag: null,
|
||||
extracted_entities: [],
|
||||
covered_entities: [],
|
||||
uncovered_entities: [],
|
||||
severity: null,
|
||||
};
|
||||
}
|
||||
|
||||
// Normal case: compute coverage
|
||||
const { covered, uncovered } = computeCoverage(extractedEntities, kbHitTexts);
|
||||
const citationScore = extractedEntities.length > 0
|
||||
? covered.length / extractedEntities.length
|
||||
: 1; // No entities to check = nothing to hallucinate about
|
||||
|
||||
// Hallucination threshold: citation_score < 0.5 for KB intents
|
||||
const hallucinationFlag = citationScore < 0.5;
|
||||
|
||||
return {
|
||||
citation_score: Math.round(citationScore * 1000) / 1000, // 3 decimal places
|
||||
hallucination_flag: hallucinationFlag,
|
||||
extracted_entities: extractedEntities,
|
||||
covered_entities: covered,
|
||||
uncovered_entities: uncovered,
|
||||
severity: hallucinationFlag ? 'low_coverage' : null,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { checkCitation, extractEntities, computeCoverage, detectFabricatedCitations };
|
||||
Reference in New Issue
Block a user