83 lines
3.9 KiB
JavaScript
83 lines
3.9 KiB
JavaScript
#!/usr/bin/env node
|
|
'use strict';
|
|
|
|
// 离线“真实模型”评测：用 fake LibreDesk 客户端跑 createAgent，不写回真实会话。
|
|
// 依赖：LLM、旧 KB (127.0.0.1:5432/products)、商品 RPC 隧道。
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const { loadConfig } = require('../bridge/src/config');
|
|
const { createAgent } = require('../bridge/src/agent');
|
|
const { createTools } = require('../bridge/src/tools');
|
|
const { createRetriever } = require('../bridge/src/rag');
|
|
|
|
const ROOT = path.resolve(__dirname, '..');
|
|
const env = {};
|
|
for (const line of fs.readFileSync(path.join(ROOT, '.env'), 'utf8').split('\n')) {
|
|
if (!line.includes('=') || line.trim().startsWith('#')) continue;
|
|
const i = line.indexOf('=');
|
|
env[line.slice(0, i).trim()] = line.slice(i + 1).trim();
|
|
}
|
|
const password = encodeURIComponent(env.POSTGRES_PASSWORD || '');
|
|
env.PG_URL = `postgres://crmnew:${password}@127.0.0.1:5440/bridge`;
|
|
env.RAG_PG_URL = 'postgres://postgres:postgres@127.0.0.1:5432/products';
|
|
const cfg = loadConfig({ ...process.env, ...env });
|
|
|
|
const CASES = [
|
|
{ id: 'policy-min-order', message: 'What is the minimum order value?', contact: { type: 'contact', email: 'eval@example.com' }, expectTool: 'search_kb', contains: ['30'] },
|
|
{ id: 'product-hoops', message: 'Do you have hoop earrings?', contact: { type: 'contact', email: 'eval@example.com' }, expectTool: 'search_products', contains: ['yehwang.com'] },
|
|
{ id: 'product-zh-rings', message: '推荐一些简约不锈钢戒指', contact: { type: 'contact', email: 'eval@example.com' }, expectTool: 'search_products', contains: ['戒指'] },
|
|
{ id: 'anon-order-gate', message: 'Where is my order?', contact: { type: 'visitor' }, containsAny: ['email', 'verify', 'verification', '邮箱', '验证'] },
|
|
];
|
|
|
|
function fakeLeadStore() {
|
|
const memory = new Map();
|
|
return {
|
|
get: async (uuid) => memory.get(uuid) || null,
|
|
save: async (uuid, slots) => { memory.set(uuid, { slots }); return { slots }; },
|
|
remove: async (uuid) => { memory.delete(uuid); },
|
|
};
|
|
}
|
|
|
|
async function runCase(c) {
|
|
let reply = '';
|
|
let metric = null;
|
|
const tools = createTools(cfg, { retriever: createRetriever(cfg) });
|
|
tools.leadStore = fakeLeadStore();
|
|
const agent = createAgent(cfg, {
|
|
tools,
|
|
client: {
|
|
getMessages: async () => [{ id: 1, text_content: c.message, sender_type: 'contact', private: false }],
|
|
sendMessage: async (_uuid, content) => { reply = content; },
|
|
getConversation: async () => ({}),
|
|
},
|
|
verification: { getVerifiedEmail: async () => null, sendCode: async () => ({}), verifyCode: async () => ({}) },
|
|
metrics: { enabled: false, record: async (m) => { metric = m; } },
|
|
contextManager: { compressHistory: async (_c, h) => h },
|
|
llmCache: { enabled: false, get: async () => null, store: async () => false },
|
|
});
|
|
const conv = { uuid: `eval-${c.id}`, inbox_id: '1', channel: 'livechat', contact: c.contact };
|
|
await agent.run(conv, { uuid: `msg-${c.id}`, text_content: c.message, content: c.message });
|
|
return { reply, metric };
|
|
}
|
|
|
|
(async () => {
|
|
let failed = 0;
|
|
for (const c of CASES) {
|
|
try {
|
|
const { reply, metric } = await runCase(c);
|
|
const tools = (metric && metric.toolCalls || []).map((t) => t.name);
|
|
const okTool = !c.expectTool || tools.includes(c.expectTool);
|
|
const okContains = !c.contains || c.contains.every((s) => reply.includes(s));
|
|
const okAny = !c.containsAny || c.containsAny.some((s) => reply.toLowerCase().includes(s.toLowerCase()));
|
|
const ok = okTool && okContains && okAny;
|
|
if (!ok) failed += 1;
|
|
console.log(`${ok ? 'PASS' : 'FAIL'} ${c.id} tools=[${tools.join(',')}] reply=${reply.slice(0, 140).replace(/\n/g, ' ')}`);
|
|
} catch (err) {
|
|
failed += 1;
|
|
console.log(`FAIL ${c.id} error=${err.message}`);
|
|
}
|
|
}
|
|
console.log(`\n${CASES.length - failed}/${CASES.length} passed`);
|
|
process.exit(failed ? 1 : 0);
|
|
})().catch((err) => { console.error('[eval-live]', err.message); process.exit(1); });
|