/**
* Comparisons & Synthesis — LLM + methodology-library tool-use loop.
*
* The agent walks its 9-step runbook with the two methodology tools
* available. The LLM is responsible for selecting comparison method,
* looking up the methodology, running the comparability check (using
* the rules carried in the methodology entry), and producing the
* result.
*/
import Anthropic from '@anthropic-ai/sdk';
import type {
Tool,
ToolUseBlock,
MessageParam,
ContentBlock,
TextBlock,
} from '@anthropic-ai/sdk/resources/messages.js';
import { z } from 'zod';
import { recordUsage } from '../../../observability/usage.js';
import { buildSystemPrompt } from './prompt.js';
import { METHODOLOGY_TOOLS, executeMethodologyTool } from '../../../intelligence/tools.js';
import {
comparisonSchema,
comparabilityFailureSchema,
type ComparisonsSynthesisInput,
type ComparisonsSynthesisOutput,
} from './schema.js';
import type { JobRequest } from '../../../types.js';
const apiKey = process.env.ANTHROPIC_API_KEY;
const client = apiKey ? new Anthropic({ apiKey }) : null;
if (!client) {
// eslint-disable-next-line no-console
console.log(`[intelligence.comparisons-synthesis] ANTHROPIC_API_KEY not set — agent will return a structured 'needs-api-key' failure.`);
}
const MODEL = 'claude-haiku-4-5';
const MAX_TOOL_ITERATIONS = 30;
const MAX_TOKENS_PER_TURN = 8000;
const ANTHROPIC_TOOLS: Tool[] = METHODOLOGY_TOOLS.map(t => ({
name: t.name,
description: t.description,
input_schema: t.input_schema,
})) as Tool[];
export const MODEL_NAME = MODEL;
export const TOOL_COUNT = METHODOLOGY_TOOLS.length;
export const TOOL_NAMES: readonly string[] = METHODOLOGY_TOOLS.map(t => t.name);
export interface LlmFailure {
readonly category: 'needs-api-key' | 'invalid-response' | 'sdk-error' | 'empty-response' | 'tool-loop-overrun';
readonly reason: string;
readonly hint?: string;
}
export type LlmResult<T> = { ok: true; value: T } | { ok: false; failure: LlmFailure };
export interface ToolCallTrace {
readonly toolName: string;
readonly input: Record<string, unknown>;
readonly ok: boolean;
readonly resultSummary: string;
readonly errorMessage?: string;
readonly at: string;
}
const responseSchema = z.object({
comparisons: z.array(comparisonSchema),
comparabilityFailures: z.array(comparabilityFailureSchema).default([]),
appliedMethodologies: z.array(z.string()).default([]),
notes: z.array(z.string()).default([]),
});
function buildUserMessage(metrics: ComparisonsSynthesisInput, job: JobRequest): string {
return [
`## Comparisons & Synthesis — runbook execution`,
``,
`## JobRequest`,
`analysisId: ${job.analysisId}`,
`question: ${job.question}`,
`entities: ${job.entities.map(e => e.id).join(', ')}`,
`targetMetrics: ${job.targetMetrics.map(m => m.key).join(', ')}`,
``,
`## Computed Metrics from Agent 2 (${metrics.metrics.length})`,
JSON.stringify(metrics.metrics, null, 2),
metrics.methodologyGaps.length > 0
? `\n## Methodology gaps from Agent 2 (${metrics.methodologyGaps.length})\n${JSON.stringify(metrics.methodologyGaps, null, 2)}`
: '',
``,
`## What to do`,
`Infer the comparisons the JobRequest implies (cross-entity peer comparisons, period-over-period trends, ratios), and for each:`,
` 1. Call find_methodologies(type="comparison_method", triggers matching the comparison shape).`,
` 2. If a methodology comes back, call get_methodology(id) to read its comparability_check.`,
` 3. Run the comparability check. If it fails, add an entry to comparabilityFailures and skip the comparison.`,
` 4. If the check passes, compute the comparison and stamp the methodology_id + inputLineage.`,
` 5. If NO methodology matches and the comparison is trivial (e.g. straight cross-entity numeric ranking on the same metric/period/unit), you may apply a built-in method with methodologyId=null and explain in methodologyRationale.`,
` 6. Where statistical context is available (mean, std dev, rank, significance), populate statisticalContext.`,
``,
`## Output`,
`Return ONLY a JSON object — no prose, no markdown fence — in this exact shape:`,
`{`,
` "comparisons": [`,
` {`,
` "comparisonId": string, // e.g. "peer-revenue-FY-2024"`,
` "method": string, // "period-over-period" | "peer-benchmark" | "ratio" | "rank"`,
` "methodologyId": string | null,`,
` "methodologyRationale": string,`,
` "entitiesCompared": string[],`,
` "periodsCovered": string[],`,
` "metricKey": string,`,
` "result": object, // method-specific shape, free`,
` "comparabilityCheckStatus": "passed" | "flagged" | "failed",`,
` "comparabilityNotes": string[],`,
` "statisticalContext": { "mean"?: number, "stdDev"?: number, "rank"?: number, "significant"?: boolean, "notes": string[] },`,
` "inputLineage": string[],`,
` "confidence": number,`,
` "flags": string[]`,
` }`,
` ],`,
` "comparabilityFailures": [`,
` { "comparisonId": string, "attempted": string, "reason": string }`,
` ],`,
` "appliedMethodologies": string[],`,
` "notes": string[]`,
`}`,
].filter(Boolean).join('\n');
}
function jsonResponseFromText(text: string): unknown {
const cleaned = text.replace(/^```(?:json)?\s*/i, '').replace(/```\s*$/i, '').trim();
try { return JSON.parse(cleaned); } catch { /* fall through */ }
const m = cleaned.match(/\{[\s\S]*\}/);
if (!m) return null;
try { return JSON.parse(m[0]); } catch { return null; }
}
function summarize(name: string, ok: boolean, result: unknown): string {
if (!ok) return 'error';
if (name === 'find_methodologies' && Array.isArray(result)) {
return `${result.length} match(es): ${result.slice(0, 5).map((r: any) => r.methodology_id).join(', ')}`;
}
if (name === 'get_methodology' && result && typeof result === 'object') {
const r = result as { methodology_id?: string; name?: string };
return `${r.methodology_id ?? '?'} — ${r.name ?? ''}`;
}
return 'ok';
}
export interface CompareResult {
readonly comparisons: ComparisonsSynthesisOutput;
readonly toolCalls: readonly ToolCallTrace[];
}
export async function buildComparisons(
metrics: ComparisonsSynthesisInput,
job: JobRequest,
onToolCall?: (t: ToolCallTrace) => void,
): Promise<LlmResult<CompareResult>> {
if (!client) {
return {
ok: false,
failure: { category: 'needs-api-key', reason: 'Comparisons & Synthesis requires the LLM but ANTHROPIC_API_KEY is not configured.' },
};
}
const system = buildSystemPrompt();
const messages: MessageParam[] = [{ role: 'user', content: buildUserMessage(metrics, job) }];
const toolCalls: ToolCallTrace[] = [];
let finalText = '';
for (let iter = 0; iter < MAX_TOOL_ITERATIONS; iter++) {
let resp;
try {
resp = await client.messages.create({
model: MODEL,
max_tokens: MAX_TOKENS_PER_TURN,
system,
tools: ANTHROPIC_TOOLS,
messages,
});
} catch (err) {
return { ok: false, failure: { category: 'sdk-error', reason: err instanceof Error ? err.message : String(err) } };
}
recordUsage('intelligence.comparisons-synthesis', MODEL, resp.usage.input_tokens, resp.usage.output_tokens);
messages.push({ role: 'assistant', content: resp.content as ContentBlock[] });
if (resp.stop_reason !== 'tool_use') {
const textBlock = resp.content.find((b): b is TextBlock => b.type === 'text');
finalText = textBlock ? textBlock.text : '';
break;
}
const toolUses = resp.content.filter((b): b is ToolUseBlock => b.type === 'tool_use');
const toolResults: { type: 'tool_result'; tool_use_id: string; content: string; is_error?: boolean }[] = [];
for (const tu of toolUses) {
const r = await executeMethodologyTool(tu.name, tu.input);
const trace: ToolCallTrace = {
toolName: tu.name,
input: (tu.input ?? {}) as Record<string, unknown>,
ok: r.ok,
resultSummary: summarize(tu.name, r.ok, r.result),
errorMessage: r.error?.message,
at: new Date().toISOString(),
};
toolCalls.push(trace);
onToolCall?.(trace);
toolResults.push({
type: 'tool_result',
tool_use_id: tu.id,
content: r.ok ? JSON.stringify(r.result) : JSON.stringify({ error: r.error }),
is_error: !r.ok,
});
}
messages.push({ role: 'user', content: toolResults });
}
if (!finalText) {
return { ok: false, failure: { category: 'tool-loop-overrun', reason: `Tool-use loop exceeded ${MAX_TOOL_ITERATIONS} iterations.` } };
}
const parsed = responseSchema.safeParse(jsonResponseFromText(finalText));
if (!parsed.success) {
return { ok: false, failure: { category: 'invalid-response', reason: `final response did not match Comparisons schema: ${parsed.error.message}` } };
}
return { ok: true, value: { comparisons: parsed.data, toolCalls } };
}