125 lines
3.9 KiB
JavaScript
125 lines
3.9 KiB
JavaScript
'use strict';
|
||
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
const vm = require('node:vm');
|
||
|
||
function stripHtml(value) {
|
||
return String(value || '')
|
||
.replace(/<br\s*\/?>/gi, '\n')
|
||
.replace(/<[^>]+>/g, ' ')
|
||
.replace(/ /g, ' ')
|
||
.replace(/&/g, '&')
|
||
.replace(/</g, '<')
|
||
.replace(/>/g, '>')
|
||
.replace(/"/g, '"')
|
||
.replace(/\s+/g, ' ')
|
||
.trim();
|
||
}
|
||
|
||
function extractFaqFromLdJson(html, max = 50) {
|
||
const re = /<script[^>]*type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
|
||
let m;
|
||
while ((m = re.exec(String(html)))) {
|
||
try {
|
||
const data = JSON.parse(m[1].trim());
|
||
const candidates = Array.isArray(data) ? data : [data];
|
||
for (const item of candidates) {
|
||
const main = item && (item.mainEntity || (item['@type'] === 'FAQPage' && item.mainEntity));
|
||
if (!Array.isArray(main)) continue;
|
||
const out = main
|
||
.filter((q) => q && q.name && q.acceptedAnswer && q.acceptedAnswer.text)
|
||
.map((q) => ({ q: stripHtml(q.name), a: stripHtml(q.acceptedAnswer.text) }))
|
||
.filter((q) => q.q && q.a)
|
||
.slice(0, max);
|
||
if (out.length) return out;
|
||
}
|
||
} catch {
|
||
// 非 JSON-LD 块忽略
|
||
}
|
||
}
|
||
return [];
|
||
}
|
||
|
||
// Nuxt SSR 页面（如 dropshipping.yehwang.com）没有 FAQPage JSON-LD，
|
||
// FAQ 数据内嵌在 window.__NUXT__ 的 payload 里（data[].info.content = [{title, content(HTML)}]）。
|
||
// 在隔离 sandbox 里求值这段自执行脚本，只取回返回对象，不接触宿主全局。
|
||
function extractNuxtPayload(html) {
|
||
const m = String(html || '').match(/__NUXT__=([\s\S]*?)<\/script>/);
|
||
if (!m) return null;
|
||
try {
|
||
return vm.runInNewContext(m[1].replace(/;\s*$/, ''), Object.create(null), { timeout: 2000 });
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function extractFaqFromNuxt(html, max = 50) {
|
||
const payload = extractNuxtPayload(html);
|
||
if (!payload || !payload.data) return [];
|
||
const out = [];
|
||
const seen = new Set();
|
||
const walk = (node) => {
|
||
if (out.length >= max) return;
|
||
if (Array.isArray(node)) {
|
||
for (const item of node) walk(item);
|
||
return;
|
||
}
|
||
if (!node || typeof node !== 'object') return;
|
||
if (typeof node.title === 'string' && typeof node.content === 'string') {
|
||
const q = stripHtml(node.title);
|
||
const a = stripHtml(node.content);
|
||
if (q && a && !seen.has(q)) {
|
||
seen.add(q);
|
||
out.push({ q, a });
|
||
}
|
||
}
|
||
for (const value of Object.values(node)) {
|
||
if (value && typeof value === 'object') walk(value);
|
||
}
|
||
};
|
||
walk(payload.data);
|
||
return out;
|
||
}
|
||
|
||
function loadFaqConfig(file) {
|
||
try {
|
||
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
||
} catch {
|
||
return { enabled: false, pages: [] };
|
||
}
|
||
}
|
||
|
||
function createFaqFetcher(config = {}) {
|
||
const pages = config.pages || [];
|
||
const timeoutMs = config.timeoutMs || 15000;
|
||
const max = config.maxFaqsPerPage || 50;
|
||
|
||
async function fetchPage(page) {
|
||
const res = await fetch(page.url, {
|
||
headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36', Accept: 'text/html,application/xhtml+xml' },
|
||
signal: AbortSignal.timeout(timeoutMs),
|
||
});
|
||
if (!res.ok) throw new Error(`fetch ${page.url} failed: HTTP ${res.status}`);
|
||
const html = await res.text();
|
||
return page.extractor === 'nuxt' ? extractFaqFromNuxt(html, max) : extractFaqFromLdJson(html, max);
|
||
}
|
||
|
||
async function fetchAll() {
|
||
const out = [];
|
||
for (const page of pages) {
|
||
try {
|
||
const questions = await fetchPage(page);
|
||
out.push({ ...page, questions });
|
||
} catch (err) {
|
||
out.push({ ...page, questions: [], error: err.message });
|
||
}
|
||
}
|
||
return out;
|
||
}
|
||
|
||
return { fetchPage, fetchAll, pages };
|
||
}
|
||
|
||
module.exports = { createFaqFetcher, loadFaqConfig, extractFaqFromLdJson, extractFaqFromNuxt, extractNuxtPayload, stripHtml };
|