371 lines
12 KiB
JavaScript
371 lines
12 KiB
JavaScript
/**
|
|
* Analyze validation results and generate tuning recommendations
|
|
* Usage: node scripts/analyze-results.js validation-report-*.json
|
|
*/
|
|
|
|
import { readFile } from 'fs/promises';
|
|
import { writeFile } from 'fs/promises';
|
|
|
|
/**
|
|
* Analyze a validation report and generate recommendations
|
|
*/
|
|
function analyzeReport(report) {
|
|
const { results, successful, failed, total } = report;
|
|
|
|
const analysis = {
|
|
summary: {
|
|
total,
|
|
successful,
|
|
failed,
|
|
successRate: report.successRate,
|
|
avgRecencyScore: report.avgRecencyScore || 0,
|
|
avgRelevanceScore: report.avgRelevanceScore || 0
|
|
},
|
|
categories: {
|
|
excellent: [], // Recent, relevant, good volume
|
|
good: [], // Some recent, mostly relevant
|
|
needsTuning: [], // Low recency or relevance
|
|
failing: [] // No results
|
|
},
|
|
recommendations: []
|
|
};
|
|
|
|
// Categorize each alert
|
|
results.forEach(result => {
|
|
if (!result.success) {
|
|
analysis.categories.failing.push(result);
|
|
return;
|
|
}
|
|
|
|
const recentRatio = result.resultCount > 0 ? result.recentCount / result.resultCount : 0;
|
|
const relevantRatio = result.resultCount > 0 ? result.relevantCount / result.resultCount : 0;
|
|
|
|
if (result.recentCount >= 3 && relevantRatio >= 0.6 && result.resultCount >= 5) {
|
|
analysis.categories.excellent.push(result);
|
|
} else if (result.recentCount >= 1 && relevantRatio >= 0.4) {
|
|
analysis.categories.good.push(result);
|
|
} else {
|
|
analysis.categories.needsTuning.push(result);
|
|
}
|
|
});
|
|
|
|
// Generate specific recommendations
|
|
|
|
// No recent results
|
|
const noRecent = results.filter(r => r.success && (r.recentCount || 0) === 0);
|
|
if (noRecent.length > 0) {
|
|
analysis.recommendations.push({
|
|
category: 'Recency Issues',
|
|
severity: 'high',
|
|
count: noRecent.length,
|
|
alerts: noRecent.map(r => r.name),
|
|
issue: 'No results from today or this week',
|
|
suggestions: [
|
|
'Broaden keywords to capture more general discussions',
|
|
'Check if topic is actively discussed (may be seasonal)',
|
|
'Consider adding trending terms related to the topic',
|
|
'Remove overly specific technical terms'
|
|
]
|
|
});
|
|
}
|
|
|
|
// Low relevance
|
|
const lowRelevance = results.filter(r => r.success && r.relevantCount < (r.resultCount / 2));
|
|
if (lowRelevance.length > 0) {
|
|
analysis.recommendations.push({
|
|
category: 'Relevance Issues',
|
|
severity: 'medium',
|
|
count: lowRelevance.length,
|
|
alerts: lowRelevance.map(r => r.name),
|
|
issue: 'Less than 50% of results are relevant',
|
|
suggestions: [
|
|
'Add more specific repair-related keywords',
|
|
'Include domain filters (site:reddit.com, site:kijiji.ca)',
|
|
'Add negative keywords to exclude noise (-job -jobs -career)',
|
|
'Use exact phrase matching with quotes for key terms'
|
|
]
|
|
});
|
|
}
|
|
|
|
// Few results
|
|
const fewResults = results.filter(r => r.success && r.resultCount < 5);
|
|
if (fewResults.length > 0) {
|
|
analysis.recommendations.push({
|
|
category: 'Low Volume',
|
|
severity: 'medium',
|
|
count: fewResults.length,
|
|
alerts: fewResults.map(r => r.name),
|
|
issue: 'Fewer than 5 results returned',
|
|
suggestions: [
|
|
'Use broader search terms (remove some specific keywords)',
|
|
'Try OR operators to include synonyms',
|
|
'Expand geographic scope',
|
|
'Check for typos in query'
|
|
]
|
|
});
|
|
}
|
|
|
|
// Failing alerts
|
|
if (failed > 0) {
|
|
const failingAlerts = results.filter(r => !r.success);
|
|
analysis.recommendations.push({
|
|
category: 'Failing Alerts',
|
|
severity: 'critical',
|
|
count: failed,
|
|
alerts: failingAlerts.map(r => r.name),
|
|
issue: 'Queries returning no results or errors',
|
|
suggestions: [
|
|
'Test query directly in Google Search',
|
|
'Simplify query structure',
|
|
'Check for syntax errors',
|
|
'Verify site filters are correct',
|
|
'Consider if topic exists in target locations'
|
|
]
|
|
});
|
|
}
|
|
|
|
return analysis;
|
|
}
|
|
|
|
/**
|
|
* Generate markdown report from analysis
|
|
*/
|
|
function generateMarkdownReport(analysis, reportName) {
|
|
const lines = [];
|
|
|
|
lines.push(`# Validation Analysis Report`);
|
|
lines.push(``);
|
|
lines.push(`**Source:** ${reportName}`);
|
|
lines.push(`**Generated:** ${new Date().toLocaleString()}`);
|
|
lines.push(``);
|
|
lines.push(`---`);
|
|
lines.push(``);
|
|
|
|
// Summary
|
|
lines.push(`## Summary`);
|
|
lines.push(``);
|
|
lines.push(`- **Total Alerts Tested:** ${analysis.summary.total}`);
|
|
lines.push(`- **Successful:** ${analysis.summary.successful} (${Math.round(analysis.summary.successRate)}%)`);
|
|
lines.push(`- **Failed:** ${analysis.summary.failed}`);
|
|
lines.push(`- **Avg Recency Score:** ${analysis.summary.avgRecencyScore}/10`);
|
|
lines.push(`- **Avg Relevance Score:** ${analysis.summary.avgRelevanceScore}`);
|
|
lines.push(``);
|
|
|
|
// Categories
|
|
lines.push(`## Alert Performance Categories`);
|
|
lines.push(``);
|
|
|
|
lines.push(`### ✅ Excellent (${analysis.categories.excellent.length})`);
|
|
lines.push(`*Recent results, high relevance, good volume*`);
|
|
lines.push(``);
|
|
if (analysis.categories.excellent.length > 0) {
|
|
analysis.categories.excellent.forEach(alert => {
|
|
lines.push(`- **${alert.name}**`);
|
|
lines.push(` - Results: ${alert.resultCount}, Recent: ${alert.recentCount}, Relevant: ${alert.relevantCount}`);
|
|
lines.push(` - **Action:** Keep as-is, this alert is performing well`);
|
|
lines.push(``);
|
|
});
|
|
} else {
|
|
lines.push(`*No alerts in this category*`);
|
|
lines.push(``);
|
|
}
|
|
|
|
lines.push(`### ✓ Good (${analysis.categories.good.length})`);
|
|
lines.push(`*Acceptable performance with room for improvement*`);
|
|
lines.push(``);
|
|
if (analysis.categories.good.length > 0) {
|
|
analysis.categories.good.forEach(alert => {
|
|
lines.push(`- **${alert.name}**`);
|
|
lines.push(` - Results: ${alert.resultCount}, Recent: ${alert.recentCount || 0}, Relevant: ${alert.relevantCount || 0}`);
|
|
lines.push(` - **Action:** Monitor and optionally tune for better results`);
|
|
lines.push(``);
|
|
});
|
|
} else {
|
|
lines.push(`*No alerts in this category*`);
|
|
lines.push(``);
|
|
}
|
|
|
|
lines.push(`### ⚠️ Needs Tuning (${analysis.categories.needsTuning.length})`);
|
|
lines.push(`*Low recency, relevance, or volume issues*`);
|
|
lines.push(``);
|
|
if (analysis.categories.needsTuning.length > 0) {
|
|
analysis.categories.needsTuning.forEach(alert => {
|
|
lines.push(`- **${alert.name}**`);
|
|
lines.push(` - Results: ${alert.resultCount}, Recent: ${alert.recentCount || 0}, Relevant: ${alert.relevantCount || 0}`);
|
|
lines.push(` - Recency Score: ${alert.avgRecencyScore || 0}/10, Relevance Score: ${alert.avgRelevanceScore || 0}`);
|
|
lines.push(` - **Action:** Requires tuning - see recommendations below`);
|
|
lines.push(``);
|
|
});
|
|
} else {
|
|
lines.push(`*No alerts in this category*`);
|
|
lines.push(``);
|
|
}
|
|
|
|
lines.push(`### ❌ Failing (${analysis.categories.failing.length})`);
|
|
lines.push(`*No results or errors*`);
|
|
lines.push(``);
|
|
if (analysis.categories.failing.length > 0) {
|
|
analysis.categories.failing.forEach(alert => {
|
|
lines.push(`- **${alert.name}**`);
|
|
lines.push(` - Error: ${alert.error || 'No results found'}`);
|
|
lines.push(` - **Action:** Critical - needs immediate attention`);
|
|
lines.push(``);
|
|
});
|
|
} else {
|
|
lines.push(`*No alerts in this category*`);
|
|
lines.push(``);
|
|
}
|
|
|
|
// Recommendations
|
|
lines.push(`---`);
|
|
lines.push(``);
|
|
lines.push(`## Tuning Recommendations`);
|
|
lines.push(``);
|
|
|
|
if (analysis.recommendations.length === 0) {
|
|
lines.push(`🎉 **All alerts are performing well! No tuning needed.**`);
|
|
lines.push(``);
|
|
} else {
|
|
analysis.recommendations.forEach((rec, idx) => {
|
|
const severityEmoji = {
|
|
critical: '🔴',
|
|
high: '🟠',
|
|
medium: '🟡',
|
|
low: '🟢'
|
|
}[rec.severity] || '⚪';
|
|
|
|
lines.push(`### ${severityEmoji} ${rec.category} (${rec.count} alerts)`);
|
|
lines.push(``);
|
|
lines.push(`**Issue:** ${rec.issue}`);
|
|
lines.push(``);
|
|
lines.push(`**Affected Alerts:**`);
|
|
rec.alerts.forEach(name => lines.push(`- ${name}`));
|
|
lines.push(``);
|
|
lines.push(`**Suggestions:**`);
|
|
rec.suggestions.forEach(suggestion => lines.push(`- ${suggestion}`));
|
|
lines.push(``);
|
|
});
|
|
}
|
|
|
|
// Priority Actions
|
|
lines.push(`---`);
|
|
lines.push(``);
|
|
lines.push(`## Priority Actions`);
|
|
lines.push(``);
|
|
|
|
const criticalRecs = analysis.recommendations.filter(r => r.severity === 'critical');
|
|
const highRecs = analysis.recommendations.filter(r => r.severity === 'high');
|
|
|
|
if (criticalRecs.length > 0) {
|
|
lines.push(`### 1. Critical Issues (Do First)`);
|
|
lines.push(``);
|
|
criticalRecs.forEach(rec => {
|
|
lines.push(`- **${rec.category}:** ${rec.count} alerts`);
|
|
lines.push(` - ${rec.suggestions[0]}`);
|
|
});
|
|
lines.push(``);
|
|
}
|
|
|
|
if (highRecs.length > 0) {
|
|
lines.push(`### 2. High Priority`);
|
|
lines.push(``);
|
|
highRecs.forEach(rec => {
|
|
lines.push(`- **${rec.category}:** ${rec.count} alerts`);
|
|
lines.push(` - ${rec.suggestions[0]}`);
|
|
});
|
|
lines.push(``);
|
|
}
|
|
|
|
const mediumRecs = analysis.recommendations.filter(r => r.severity === 'medium');
|
|
if (mediumRecs.length > 0) {
|
|
lines.push(`### 3. Medium Priority (Tune When Possible)`);
|
|
lines.push(``);
|
|
mediumRecs.forEach(rec => {
|
|
lines.push(`- **${rec.category}:** ${rec.count} alerts`);
|
|
});
|
|
lines.push(``);
|
|
}
|
|
|
|
// Next Steps
|
|
lines.push(`---`);
|
|
lines.push(``);
|
|
lines.push(`## Next Steps`);
|
|
lines.push(``);
|
|
lines.push(`1. **Review failing alerts first** - Fix syntax errors or verify topic exists`);
|
|
lines.push(`2. **Address recency issues** - Broaden keywords for alerts with no recent results`);
|
|
lines.push(`3. **Improve relevance** - Add filters and negative keywords`);
|
|
lines.push(`4. **Re-test after changes** - Run validation again to verify improvements`);
|
|
lines.push(`5. **Keep excellent alerts as-is** - Don't fix what isn't broken`);
|
|
lines.push(``);
|
|
|
|
return lines.join('\n');
|
|
}
|
|
|
|
/**
|
|
* Main function
|
|
*/
|
|
async function main() {
|
|
const args = process.argv.slice(2);
|
|
|
|
if (args.length === 0) {
|
|
console.log(`
|
|
Usage:
|
|
node scripts/analyze-results.js <report-file.json>
|
|
|
|
Example:
|
|
node scripts/analyze-results.js validation-report-1699999999999.json
|
|
`);
|
|
process.exit(0);
|
|
}
|
|
|
|
const reportFile = args[0];
|
|
|
|
try {
|
|
console.log(`\n📊 Analyzing report: ${reportFile}\n`);
|
|
|
|
// Read report
|
|
const reportData = await readFile(reportFile, 'utf-8');
|
|
const report = JSON.parse(reportData);
|
|
|
|
// Analyze
|
|
const analysis = analyzeReport(report);
|
|
|
|
// Generate markdown report
|
|
const markdown = generateMarkdownReport(analysis, reportFile);
|
|
|
|
// Save analysis
|
|
const analysisFile = reportFile.replace('.json', '-analysis.md');
|
|
await writeFile(analysisFile, markdown);
|
|
|
|
// Print summary
|
|
console.log(`✅ Analysis complete!\n`);
|
|
console.log(`📈 Performance Summary:`);
|
|
console.log(` Excellent: ${analysis.categories.excellent.length}`);
|
|
console.log(` Good: ${analysis.categories.good.length}`);
|
|
console.log(` Needs Tuning: ${analysis.categories.needsTuning.length}`);
|
|
console.log(` Failing: ${analysis.categories.failing.length}\n`);
|
|
|
|
if (analysis.recommendations.length > 0) {
|
|
console.log(`🔧 ${analysis.recommendations.length} recommendation(s) generated\n`);
|
|
analysis.recommendations.forEach(rec => {
|
|
console.log(` ${rec.category}: ${rec.count} alerts (${rec.severity})`);
|
|
});
|
|
console.log(``);
|
|
}
|
|
|
|
console.log(`💾 Full analysis saved to: ${analysisFile}\n`);
|
|
|
|
} catch (error) {
|
|
console.error(`\n❌ Error: ${error.message}\n`);
|
|
process.exit(1);
|
|
}
|
|
}
|
|
|
|
// Run if called directly
|
|
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
main().catch(console.error);
|
|
}
|
|
|
|
export { analyzeReport, generateMarkdownReport };
|
|
|