feat: port quality, escalation, KB version, site-map KB, experiments and metrics
This commit is contained in:
wangliang
@@ -0,0 +1,235 @@
|
||||
const { describe, it } = require('node:test');
|
||||
const assert = require('node:assert');
|
||||
const { checkCitation, extractEntities, computeCoverage, detectFabricatedCitations } = require('../src/citation-checker');
|
||||
|
||||
describe('citation_checker', () => {
|
||||
describe('extractEntities', () => {
|
||||
it('extracts EUR amounts with € symbol', () => {
|
||||
const entities = extractEntities('Free shipping for orders over €200.');
|
||||
const amountEntities = entities.filter(e => e.startsWith('amount:'));
|
||||
assert.ok(amountEntities.some(e => e === 'amount:200'), `Expected amount:200, got ${amountEntities}`);
|
||||
});
|
||||
|
||||
it('extracts EUR amounts with EUR prefix', () => {
|
||||
const entities = extractEntities('The minimum order is EUR 30.');
|
||||
const amountEntities = entities.filter(e => e.startsWith('amount:'));
|
||||
assert.ok(amountEntities.some(e => e === 'amount:30'), `Expected amount:30, got ${amountEntities}`);
|
||||
});
|
||||
|
||||
it('extracts EUR amounts with suffix', () => {
|
||||
const entities = extractEntities('Total cost: 150€');
|
||||
const amountEntities = entities.filter(e => e.startsWith('amount:'));
|
||||
assert.ok(amountEntities.some(e => e === 'amount:150'), `Expected amount:150, got ${amountEntities}`);
|
||||
});
|
||||
|
||||
it('extracts EUR amounts with "euros" suffix', () => {
|
||||
const entities = extractEntities('Shipping costs 5 euros.');
|
||||
const amountEntities = entities.filter(e => e.startsWith('amount:'));
|
||||
assert.ok(amountEntities.some(e => e === 'amount:5'), `Expected amount:5, got ${amountEntities}`);
|
||||
});
|
||||
|
||||
it('extracts percentages', () => {
|
||||
const entities = extractEntities('We offer a 10% discount for members.');
|
||||
const pctEntities = entities.filter(e => e.startsWith('pct:'));
|
||||
assert.ok(pctEntities.some(e => e === 'pct:10'), `Expected pct:10, got ${pctEntities}`);
|
||||
});
|
||||
|
||||
it('extracts time limits in days', () => {
|
||||
const entities = extractEntities('You have 30 days to return items.');
|
||||
const timeEntities = entities.filter(e => e.startsWith('time:'));
|
||||
assert.ok(timeEntities.length > 0, `Expected time entities, got ${entities}`);
|
||||
});
|
||||
|
||||
it('extracts time limits in months', () => {
|
||||
const entities = extractEntities('Warranty lasts 6 months.');
|
||||
const timeEntities = entities.filter(e => e.startsWith('time:'));
|
||||
assert.ok(timeEntities.length > 0, `Expected time entities, got ${entities}`);
|
||||
});
|
||||
|
||||
it('extracts EU country names', () => {
|
||||
const entities = extractEntities('We ship to all EU countries including Netherlands and Germany.');
|
||||
const countryEntities = entities.filter(e => e.startsWith('country:'));
|
||||
assert.ok(countryEntities.some(e => e === 'country:netherlands'), `Expected country:netherlands, got ${countryEntities}`);
|
||||
assert.ok(countryEntities.some(e => e === 'country:germany'), `Expected country:germany, got ${countryEntities}`);
|
||||
});
|
||||
|
||||
it('extracts English policy terms', () => {
|
||||
const entities = extractEntities('Our refund and return policy covers shipping.');
|
||||
const policyEntities = entities.filter(e => e.startsWith('policy:'));
|
||||
assert.ok(policyEntities.some(e => e === 'policy:refund'), `Expected policy:refund, got ${policyEntities}`);
|
||||
assert.ok(policyEntities.some(e => e === 'policy:return'), `Expected policy:return, got ${policyEntities}`);
|
||||
assert.ok(policyEntities.some(e => e === 'policy:shipping'), `Expected policy:shipping, got ${policyEntities}`);
|
||||
});
|
||||
|
||||
it('extracts number+unit combos', () => {
|
||||
const entities = extractEntities('Minimum order is 5 pieces.');
|
||||
const unitEntities = entities.filter(e => e.startsWith('unit:'));
|
||||
assert.ok(unitEntities.length > 0, `Expected unit entities, got ${entities}`);
|
||||
});
|
||||
|
||||
it('returns empty array for null/undefined text', () => {
|
||||
assert.deepStrictEqual(extractEntities(null), []);
|
||||
assert.deepStrictEqual(extractEntities(undefined), []);
|
||||
assert.deepStrictEqual(extractEntities(''), []);
|
||||
});
|
||||
|
||||
it('handles decimal amounts', () => {
|
||||
const entities = extractEntities('The fee is €4.95.');
|
||||
const amountEntities = entities.filter(e => e.startsWith('amount:'));
|
||||
assert.ok(amountEntities.some(e => e === 'amount:4.95'), `Expected amount:4.95, got ${amountEntities}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('computeCoverage', () => {
|
||||
it('finds covered entities in KB texts', () => {
|
||||
const replyEntities = ['amount:200', 'policy:shipping'];
|
||||
const kbHitTexts = ['Free shipping for orders over €200 within Netherlands.'];
|
||||
const { covered, uncovered } = computeCoverage(replyEntities, kbHitTexts);
|
||||
assert.ok(covered.includes('amount:200'), `Expected amount:200 in covered, got ${covered}`);
|
||||
assert.ok(covered.includes('policy:shipping'), `Expected policy:shipping in covered, got ${covered}`);
|
||||
assert.strictEqual(uncovered.length, 0);
|
||||
});
|
||||
|
||||
it('identifies uncovered entities', () => {
|
||||
const replyEntities = ['amount:500', 'policy:shipping', 'country:spain'];
|
||||
const kbHitTexts = ['Free shipping for orders over €200 within Netherlands.'];
|
||||
const { covered, uncovered } = computeCoverage(replyEntities, kbHitTexts);
|
||||
assert.ok(covered.includes('policy:shipping'), `Expected policy:shipping in covered`);
|
||||
assert.ok(uncovered.includes('amount:500'), `Expected amount:500 in uncovered`);
|
||||
assert.ok(uncovered.includes('country:spain'), `Expected country:spain in uncovered`);
|
||||
});
|
||||
|
||||
it('handles KB hit objects with question/answer', () => {
|
||||
const replyEntities = ['amount:200', 'policy:shipping'];
|
||||
const kbHitTexts = [{ question: 'Shipping policy?', answer: 'Free shipping for orders over €200.', category: 'shipping' }];
|
||||
const { covered } = computeCoverage(replyEntities, kbHitTexts);
|
||||
assert.ok(covered.includes('amount:200'));
|
||||
});
|
||||
|
||||
it('returns empty for empty reply entities', () => {
|
||||
const { covered, uncovered } = computeCoverage([], ['Some KB text']);
|
||||
assert.deepStrictEqual(covered, []);
|
||||
assert.deepStrictEqual(uncovered, []);
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkCitation', () => {
|
||||
it('returns null for non-KB intent (product_search)', () => {
|
||||
const result = checkCitation('Some product reply', ['KB text'], 'product_search');
|
||||
assert.strictEqual(result.citation_score, null);
|
||||
assert.strictEqual(result.hallucination_flag, null);
|
||||
});
|
||||
|
||||
it('returns null for non-KB intent (order_tracking)', () => {
|
||||
const result = checkCitation('Your order is shipped', [], 'order_tracking');
|
||||
assert.strictEqual(result.citation_score, null);
|
||||
assert.strictEqual(result.hallucination_flag, null);
|
||||
});
|
||||
|
||||
it('returns null for non-KB intent (inventory_check)', () => {
|
||||
const result = checkCitation('5 items in stock', [], 'inventory_check');
|
||||
assert.strictEqual(result.citation_score, null);
|
||||
assert.strictEqual(result.hallucination_flag, null);
|
||||
});
|
||||
|
||||
it('computes high citation_score for well-supported reply', () => {
|
||||
const reply = 'Free shipping for orders over €200 within the Netherlands. We offer a 30-day return policy.';
|
||||
const kbHits = [{ question: 'Shipping?', answer: 'Free shipping for orders over €200 within Netherlands.', category: 'shipping' }];
|
||||
const result = checkCitation(reply, kbHits, 'policy_faq');
|
||||
assert.ok(result.citation_score >= 0.5, `Expected score >= 0.5, got ${result.citation_score}`);
|
||||
assert.strictEqual(result.hallucination_flag, false);
|
||||
});
|
||||
|
||||
it('flags hallucination for low coverage reply', () => {
|
||||
const reply = 'Free shipping for orders over €200 to ALL EU countries including Spain and Italy. 50% discount on membership.';
|
||||
const kbHits = ['Free shipping for orders over €200 within Netherlands only. No special discounts.'];
|
||||
const result = checkCitation(reply, kbHits, 'policy_faq');
|
||||
assert.strictEqual(result.hallucination_flag, true);
|
||||
assert.ok(result.citation_score < 0.5, `Expected score < 0.5, got ${result.citation_score}`);
|
||||
assert.strictEqual(result.severity, 'low_coverage');
|
||||
});
|
||||
|
||||
it('returns citation_score=0 and hallucination_flag=true for no KB hits with policy entities', () => {
|
||||
const reply = 'Our refund policy allows returns within 30 days. Shipping costs €5.99.';
|
||||
const result = checkCitation(reply, [], 'policy_faq');
|
||||
assert.strictEqual(result.citation_score, 0);
|
||||
assert.strictEqual(result.hallucination_flag, true);
|
||||
assert.strictEqual(result.severity, 'no_kb_support');
|
||||
});
|
||||
|
||||
it('returns null for no KB hits and no factual entities', () => {
|
||||
const reply = 'Hello! How can I help you today?';
|
||||
const result = checkCitation(reply, [], 'general');
|
||||
assert.strictEqual(result.citation_score, null);
|
||||
assert.strictEqual(result.hallucination_flag, null);
|
||||
});
|
||||
|
||||
it('returns null for null/empty reply text', () => {
|
||||
const result = checkCitation(null, ['KB text'], 'policy_faq');
|
||||
assert.strictEqual(result.citation_score, null);
|
||||
assert.strictEqual(result.hallucination_flag, null);
|
||||
});
|
||||
|
||||
it('returns citation_score=1 when no entities are extracted from reply', () => {
|
||||
const reply = 'Yes, that is correct.';
|
||||
const kbHits = ['Some KB answer about something.'];
|
||||
const result = checkCitation(reply, kbHits, 'policy_faq');
|
||||
// No entities = nothing to hallucinate about
|
||||
assert.strictEqual(result.citation_score, 1);
|
||||
assert.strictEqual(result.hallucination_flag, false);
|
||||
});
|
||||
|
||||
it('handles general intent with KB hits', () => {
|
||||
const reply = 'You can get a refund within 30 days.';
|
||||
const kbHits = ['Our refund policy: 30-day return window.'];
|
||||
const result = checkCitation(reply, kbHits, 'general');
|
||||
assert.ok(result.citation_score >= 0.5);
|
||||
});
|
||||
});
|
||||
|
||||
describe('detectFabricatedCitations', () => {
|
||||
it('detects cross-domain URLs not in KB sources', () => {
|
||||
const reply = 'See our policy at https://evil-competitor.com/fake-policy for details.';
|
||||
const kbSources = ['https://yehwang.com/real-policy'];
|
||||
const fabricated = detectFabricatedCitations(reply, kbSources);
|
||||
assert.strictEqual(fabricated.length, 1);
|
||||
assert.ok(fabricated[0].includes('evil-competitor.com'));
|
||||
});
|
||||
|
||||
it('allows same-origin navigation links not in KB sources (站内导航不算编造)', () => {
|
||||
const reply = 'Browse items on https://dropshipping.yehwang.com';
|
||||
const kbSources = ['https://dropshipping.yehwang.com/faq/returns'];
|
||||
const fabricated = detectFabricatedCitations(reply, kbSources);
|
||||
assert.strictEqual(fabricated.length, 0);
|
||||
});
|
||||
|
||||
it('returns empty when all reply URLs are in KB sources', () => {
|
||||
const reply = 'See https://yehwang.com/policy for details.';
|
||||
const kbSources = ['https://yehwang.com/policy'];
|
||||
const fabricated = detectFabricatedCitations(reply, kbSources);
|
||||
assert.strictEqual(fabricated.length, 0);
|
||||
});
|
||||
|
||||
it('returns empty for reply with no URLs', () => {
|
||||
const reply = 'Free shipping for orders over €200.';
|
||||
const fabricated = detectFabricatedCitations(reply, ['https://yehwang.com/policy']);
|
||||
assert.strictEqual(fabricated.length, 0);
|
||||
});
|
||||
|
||||
it('returns empty for null/undefined inputs', () => {
|
||||
assert.deepStrictEqual(detectFabricatedCitations(null, []), []);
|
||||
assert.deepStrictEqual(detectFabricatedCitations('', null), []);
|
||||
assert.deepStrictEqual(detectFabricatedCitations('', []), []);
|
||||
});
|
||||
|
||||
it('flags fabricated cross-domain citation as severe hallucination in checkCitation', () => {
|
||||
const reply = 'See our policy at https://other-site.com/fake-page for details about €200 free shipping.';
|
||||
const kbHits = ['Free shipping over €200.'];
|
||||
const kbSources = ['https://yehwang.com/real-shipping-policy'];
|
||||
const result = checkCitation(reply, kbHits, 'policy_faq', kbSources);
|
||||
assert.strictEqual(result.hallucination_flag, true);
|
||||
assert.strictEqual(result.severity, 'fabricated_citation');
|
||||
assert.strictEqual(result.citation_score, 0);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user