315 lines
12 KiB
JavaScript
315 lines
12 KiB
JavaScript
// citation_checker.js — Entity extraction + coverage calculation + hallucination flagging
|
||
// Shared between offline eval and online scoring paths
|
||
|
||
const EU_COUNTRIES = new Set([
|
||
'austria', 'belgium', 'bulgaria', 'croatia', 'cyprus', 'czech republic', 'czechia',
|
||
'denmark', 'estonia', 'finland', 'france', 'germany', 'greece', 'hungary', 'ireland',
|
||
'italy', 'latvia', 'lithuania', 'luxembourg', 'malta', 'netherlands', 'poland',
|
||
'portugal', 'romania', 'slovakia', 'slovenia', 'spain', 'sweden',
|
||
// Also common forms / abbreviations
|
||
'uk', 'united kingdom', 'england', 'scotland', 'wales', 'northern ireland',
|
||
// German names
|
||
'deutschland', 'österreich', 'schweiz',
|
||
// French names
|
||
'allemagne', 'pays-bas', 'espagne', 'italie', 'belgique',
|
||
// Dutch names
|
||
'duitsland', 'nederland', 'belgië', 'spanje', 'italië', 'frankrijk',
|
||
]);
|
||
|
||
const POLICY_TERMS_EN = [
|
||
'refund', 'return', 'shipping', 'discount', 'vat', 'membership', 'dropship',
|
||
'factory-direct', 'pre-order', 'preorder', 'backorder', 'cancellation', 'cancel',
|
||
'exchange', 'warranty', 'guarantee', 'minimum order', 'bulk order', 'wholesale',
|
||
'retail', 'reseller', 'business', 'corporate', 'invoice', 'payment', 'delivery',
|
||
'pickup', 'lead time', 'turnaround', 'restock', 'reorder', 'custom', 'personalize',
|
||
'engrave', 'sample', 'catalog', 'brochure', 'newsletter', 'subscribe', 'unsubscribe',
|
||
'account', 'register', 'login', 'sign up', 'password',
|
||
];
|
||
|
||
// Multilingual policy term translations (keyed by normalized term)
|
||
const POLICY_TERMS_I18N = {
|
||
// German
|
||
'rückerstattung': 'refund', 'rückgabe': 'return', 'versand': 'shipping',
|
||
'versandkosten': 'shipping', 'rabatt': 'discount', 'mehrwertsteuer': 'vat',
|
||
'mitgliedschaft': 'membership', 'stornierung': 'cancellation', 'stornieren': 'cancel',
|
||
'rechnung': 'invoice', 'zahlung': 'payment', 'lieferung': 'delivery',
|
||
'lieferzeit': 'delivery', 'großhandel': 'wholesale', 'katalog': 'catalog',
|
||
'registrieren': 'register', 'konto': 'account',
|
||
// French
|
||
'remboursement': 'refund', 'retour': 'return', 'expédition': 'shipping',
|
||
'frais de livraison': 'shipping', 'livraison': 'delivery', 'remise': 'discount',
|
||
'réduction': 'discount', 'tva': 'vat', 'adhésion': 'membership',
|
||
'annulation': 'cancellation', 'annuler': 'cancel', 'facture': 'invoice',
|
||
'paiement': 'payment', 'grossiste': 'wholesale', 'catalogue': 'catalog',
|
||
'inscription': 'register', 'compte': 'account',
|
||
// Dutch
|
||
'terugbetaling': 'refund', 'retour': 'return', 'retourneren': 'return',
|
||
'verzending': 'shipping', 'verzendkosten': 'shipping', 'korting': 'discount',
|
||
'btw': 'vat', 'lidmaatschap': 'membership', 'annulering': 'cancellation',
|
||
'annuleren': 'cancel', 'factuur': 'invoice', 'betaling': 'payment',
|
||
'groothandel': 'wholesale', 'catalogus': 'catalog', 'registreren': 'register',
|
||
'account': 'account',
|
||
// Italian
|
||
'rimborso': 'refund', 'reso': 'return', 'spedizione': 'shipping',
|
||
'costi di spedizione': 'shipping', 'sconto': 'discount', 'iva': 'vat',
|
||
'iscrizione': 'membership', 'cancellazione': 'cancellation', 'cancellare': 'cancel',
|
||
'fattura': 'invoice', 'pagamento': 'payment', 'ingrosso': 'wholesale',
|
||
'catalogo': 'catalog', 'registrare': 'register',
|
||
};
|
||
|
||
// Non-KB intents that should skip hallucination checking
|
||
const NON_KB_INTENTS = new Set(['product_search', 'order_tracking', 'inventory_check', 'sales_lead']);
|
||
|
||
/**
|
||
* Extract key entities from text.
|
||
* Entities: amounts (€/EUR), percentages, time limits (days/weeks/months),
|
||
* country names, policy terms, number+unit combos.
|
||
*/
|
||
function extractEntities(text) {
|
||
if (!text || typeof text !== 'string') return [];
|
||
const lower = text.toLowerCase();
|
||
const entities = new Set();
|
||
|
||
// Amounts: €200, EUR 200, 200€, 200 euros, 200 eur
|
||
const amountMatches = text.matchAll(/(?:€|EUR\s?|euros?\s?)\s?(\d+(?:[.,]\d+)?)/gi);
|
||
for (const m of amountMatches) {
|
||
entities.add(`amount:${m[1].replace(',', '.')}`);
|
||
}
|
||
const amountMatches2 = text.matchAll(/(\d+(?:[.,]\d+)?)\s?(?:€|EUR|euros?|eur)/gi);
|
||
for (const m of amountMatches2) {
|
||
entities.add(`amount:${m[1].replace(',', '.')}`);
|
||
}
|
||
|
||
// Percentages: 10%, 10 percent
|
||
const pctMatches = text.matchAll(/(\d+(?:[.,]\d+)?)\s*(?:%|percent)/gi);
|
||
for (const m of pctMatches) {
|
||
entities.add(`pct:${m[1].replace(',', '.')}`);
|
||
}
|
||
|
||
// Time limits: 30 days, 2 weeks, 3 months, 14 Tage, 30 jours, 30 dagen, 5-7 werkdagen
|
||
const timeMatches = text.matchAll(/(\d+(?:\s*[-–]\s*\d+)?)\s*(?:days?|weeks?|months?|werktage?|werkdagen?|jours?|giorni?|dagen?|Tage?|settimane?|mois)/gi);
|
||
for (const m of timeMatches) {
|
||
entities.add(`time:${m[0].toLowerCase().replace(/\s+/g, '')}`);
|
||
}
|
||
|
||
// Number + unit combos: 5kg, 3 pieces, 10 items, 2 kg, 5 stück, 3 pièces, 4 stuks
|
||
const unitMatches = text.matchAll(/(\d+(?:[.,]\d+)?)\s*(?:kg|g|lb|oz|pieces?|items?|units?|stück|pièces?|stuks?|pairs?|sets?|pcs)/gi);
|
||
for (const m of unitMatches) {
|
||
entities.add(`unit:${m[0].toLowerCase().replace(/\s+/g, '')}`);
|
||
}
|
||
|
||
// EU country names
|
||
const words = lower.split(/[\s,.;:!?"'()\-/]+/).filter(Boolean);
|
||
// Also check bigrams for "united kingdom", "czech republic"
|
||
for (let i = 0; i < words.length; i++) {
|
||
if (EU_COUNTRIES.has(words[i])) {
|
||
entities.add(`country:${words[i]}`);
|
||
}
|
||
if (i + 1 < words.length) {
|
||
const bigram = words[i] + ' ' + words[i + 1];
|
||
if (EU_COUNTRIES.has(bigram)) {
|
||
entities.add(`country:${bigram}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
// English policy terms (whole word match)
|
||
for (const term of POLICY_TERMS_EN) {
|
||
const pattern = new RegExp('\\b' + term.replace(/[- ]/g, '[- ]') + '\\b', 'i');
|
||
if (pattern.test(text)) {
|
||
entities.add(`policy:${term.toLowerCase()}`);
|
||
}
|
||
}
|
||
|
||
// Multilingual policy terms
|
||
for (const [foreignTerm, englishTerm] of Object.entries(POLICY_TERMS_I18N)) {
|
||
if (lower.includes(foreignTerm.toLowerCase())) {
|
||
entities.add(`policy:${englishTerm}`);
|
||
}
|
||
}
|
||
|
||
return [...entities];
|
||
}
|
||
|
||
/**
|
||
* Normalize entities for cross-language comparison.
|
||
* Maps multilingual entities to their English equivalents.
|
||
*/
|
||
function normalizeEntity(entity) {
|
||
// Already prefixed with type: e.g., "policy:refund"
|
||
return entity;
|
||
}
|
||
|
||
/**
|
||
* Check if entities from the reply are covered by KB hit texts.
|
||
* Returns { covered, uncovered } entity arrays.
|
||
*/
|
||
function computeCoverage(replyEntities, kbHitTexts) {
|
||
if (!replyEntities.length) return { covered: [], uncovered: [] };
|
||
|
||
// Collect all entities from KB texts
|
||
const kbEntities = new Set();
|
||
for (const hit of kbHitTexts) {
|
||
const text = typeof hit === 'string' ? hit : [hit.question, hit.answer, hit.category].filter(Boolean).join(' ');
|
||
const extracted = extractEntities(text);
|
||
for (const e of extracted) {
|
||
kbEntities.add(e);
|
||
}
|
||
}
|
||
|
||
const covered = [];
|
||
const uncovered = [];
|
||
for (const entity of replyEntities) {
|
||
if (kbEntities.has(entity)) {
|
||
covered.push(entity);
|
||
} else {
|
||
uncovered.push(entity);
|
||
}
|
||
}
|
||
|
||
return { covered, uncovered };
|
||
}
|
||
|
||
/**
|
||
* Check for fabricated citations — URLs in the reply that are NOT in the KB sources.
|
||
* Returns array of fabricated URLs found.
|
||
*/
|
||
function detectFabricatedCitations(replyText, kbSources) {
|
||
if (!replyText || !kbSources || !kbSources.length) return [];
|
||
|
||
// Extract URLs from reply
|
||
const urlMatches = replyText.matchAll(/https?:\/\/[^\s)\]>"']+/gi);
|
||
const replyUrls = new Set();
|
||
for (const m of urlMatches) {
|
||
replyUrls.add(m[0].toLowerCase().replace(/\/+$/, ''));
|
||
}
|
||
|
||
// Normalize KB source URLs
|
||
const kbUrls = new Set();
|
||
for (const url of kbSources) {
|
||
if (typeof url === 'string') {
|
||
kbUrls.add(url.toLowerCase().replace(/\/+$/, ''));
|
||
}
|
||
}
|
||
|
||
// Fabricated = URLs in reply not found in KB sources.
|
||
// 允许"与 KB 来源同源(根域名)"的链接——AI 附加的站内导航链接(如 dropshipping.yehwang.com)
|
||
// 不算编造引用；只有引用了一个完全不同的域名/来源才标记。
|
||
const kbOrigins = new Set();
|
||
for (const url of kbUrls) {
|
||
try {
|
||
kbOrigins.add(new URL(url).origin.toLowerCase());
|
||
} catch (e) { /* 忽略非法 URL */ }
|
||
}
|
||
const fabricated = [];
|
||
for (const url of replyUrls) {
|
||
if (kbUrls.has(url)) continue;
|
||
try {
|
||
const origin = new URL(url).origin.toLowerCase();
|
||
if (kbOrigins.has(origin)) continue; // 同源站内链接 → 不算编造
|
||
} catch (e) { /* 非法 URL 跳过 */ }
|
||
fabricated.push(url);
|
||
}
|
||
|
||
return fabricated;
|
||
}
|
||
|
||
/**
|
||
* Main entry point: check citation consistency of a reply against KB hits.
|
||
*
|
||
* @param {string} replyText - Agent's final reply text (stripped of HTML)
|
||
* @param {Array<string|object>} kbHitTexts - KB hit texts (strings or objects with question/answer/category)
|
||
* @param {string} intent - Current intent label
|
||
* @param {Array<string>} [kbSources] - KB source URLs (for fabricated citation detection)
|
||
* @returns {{ citation_score: number|null, hallucination_flag: boolean|null, extracted_entities: string[], covered_entities: string[], uncovered_entities: string[], severity: string|null }}
|
||
*/
|
||
function checkCitation(replyText, kbHitTexts, intent, kbSources) {
|
||
// Non-KB intents: skip hallucination checking
|
||
if (NON_KB_INTENTS.has(intent)) {
|
||
return {
|
||
citation_score: null,
|
||
hallucination_flag: null,
|
||
extracted_entities: [],
|
||
covered_entities: [],
|
||
uncovered_entities: [],
|
||
severity: null,
|
||
};
|
||
}
|
||
|
||
// No reply text: nothing to check
|
||
if (!replyText || typeof replyText !== 'string') {
|
||
return {
|
||
citation_score: null,
|
||
hallucination_flag: null,
|
||
extracted_entities: [],
|
||
covered_entities: [],
|
||
uncovered_entities: [],
|
||
severity: null,
|
||
};
|
||
}
|
||
|
||
// Check for fabricated citations first (most severe)
|
||
const fabricatedUrls = detectFabricatedCitations(replyText, kbSources || []);
|
||
if (fabricatedUrls.length > 0) {
|
||
return {
|
||
citation_score: 0,
|
||
hallucination_flag: true,
|
||
extracted_entities: [],
|
||
covered_entities: [],
|
||
uncovered_entities: [],
|
||
severity: 'fabricated_citation',
|
||
};
|
||
}
|
||
|
||
// No KB hits but KB-intent (policy_faq/general): if reply has substantive entities, flag
|
||
const hasKbHits = kbHitTexts && kbHitTexts.length > 0;
|
||
const extractedEntities = extractEntities(replyText);
|
||
|
||
if (!hasKbHits) {
|
||
// No KB data to check against — if there are policy entities, likely hallucinated
|
||
const hasPolicyEntities = extractedEntities.some(e => e.startsWith('policy:'));
|
||
const hasFactualEntities = extractedEntities.some(e => e.startsWith('amount:') || e.startsWith('pct:') || e.startsWith('time:'));
|
||
|
||
if (hasPolicyEntities || hasFactualEntities) {
|
||
return {
|
||
citation_score: 0,
|
||
hallucination_flag: true,
|
||
extracted_entities: extractedEntities,
|
||
covered_entities: [],
|
||
uncovered_entities: extractedEntities,
|
||
severity: 'no_kb_support',
|
||
};
|
||
}
|
||
|
||
// No entities extracted, no KB hits — can't determine, not flagged
|
||
return {
|
||
citation_score: null,
|
||
hallucination_flag: null,
|
||
extracted_entities: [],
|
||
covered_entities: [],
|
||
uncovered_entities: [],
|
||
severity: null,
|
||
};
|
||
}
|
||
|
||
// Normal case: compute coverage
|
||
const { covered, uncovered } = computeCoverage(extractedEntities, kbHitTexts);
|
||
const citationScore = extractedEntities.length > 0
|
||
? covered.length / extractedEntities.length
|
||
: 1; // No entities to check = nothing to hallucinate about
|
||
|
||
// Hallucination threshold: citation_score < 0.5 for KB intents
|
||
const hallucinationFlag = citationScore < 0.5;
|
||
|
||
return {
|
||
citation_score: Math.round(citationScore * 1000) / 1000, // 3 decimal places
|
||
hallucination_flag: hallucinationFlag,
|
||
extracted_entities: extractedEntities,
|
||
covered_entities: covered,
|
||
uncovered_entities: uncovered,
|
||
severity: hallucinationFlag ? 'low_coverage' : null,
|
||
};
|
||
}
|
||
|
||
module.exports = { checkCitation, extractEntities, computeCoverage, detectFabricatedCitations };
|