BID · Console
Baseline · Intelligence · Decision
src/agents/intelligence/analytical-table/llm.ts 9,351 bytes · typescript
/**
 * Analytical Table — LLM judgment + methodology-library tool-use loop
 * (Std 3 + Std 5).
 *
 * The agent gives Anthropic the two methodology-library tools
 * (find_methodologies, get_methodology) and asks the model to walk
 * its runbook: structure the Pillar 1 records into an analytical
 * table, applying any declared normalization methodologies it finds
 * in the library.
 *
 * Self-contained Anthropic SDK wrapper per the codebase convention
 * (no shared LLM layer across agents). Std 12: if ANTHROPIC_API_KEY
 * is missing, returns a structured `needs-api-key` failure — the
 * orchestrator never sees a raw exception.
 */

import Anthropic from '@anthropic-ai/sdk';
import type {
  Tool,
  ToolUseBlock,
  MessageParam,
  ContentBlock,
  TextBlock,
} from '@anthropic-ai/sdk/resources/messages.js';
import { z } from 'zod';

import { recordUsage } from '../../../observability/usage.js';
import { buildSystemPrompt } from './prompt.js';
import { METHODOLOGY_TOOLS, executeMethodologyTool } from '../../../intelligence/tools.js';
import {
  analyticalCellSchema,
  type AnalyticalTableInput,
  type AnalyticalTableOutput,
} from './schema.js';
import type { JobRequest } from '../../../types.js';

const apiKey = process.env.ANTHROPIC_API_KEY;
const client = apiKey ? new Anthropic({ apiKey }) : null;

if (!client) {
  // eslint-disable-next-line no-console
  console.log(`[intelligence.analytical-table] ANTHROPIC_API_KEY not set — agent will return a structured 'needs-api-key' failure.`);
}

const MODEL = 'claude-haiku-4-5';
const MAX_TOOL_ITERATIONS = 20;
const MAX_TOKENS_PER_TURN = 8000;

const ANTHROPIC_TOOLS: Tool[] = METHODOLOGY_TOOLS.map(t => ({
  name: t.name,
  description: t.description,
  input_schema: t.input_schema,
})) as Tool[];

export const MODEL_NAME = MODEL;
export const TOOL_COUNT = METHODOLOGY_TOOLS.length;
export const TOOL_NAMES: readonly string[] = METHODOLOGY_TOOLS.map(t => t.name);

export interface LlmFailure {
  readonly category: 'needs-api-key' | 'invalid-response' | 'sdk-error' | 'empty-response' | 'tool-loop-overrun';
  readonly reason: string;
  readonly hint?: string;
}
export type LlmResult<T> = { ok: true; value: T } | { ok: false; failure: LlmFailure };

export interface ToolCallTrace {
  readonly toolName: string;
  readonly input: Record<string, unknown>;
  readonly ok: boolean;
  readonly resultSummary: string;
  readonly errorMessage?: string;
  readonly at: string;
}

const responseSchema = z.object({
  entities: z.array(z.string()),
  metrics: z.array(z.string()),
  periods: z.array(z.string()),
  cells: z.array(analyticalCellSchema),
  missingCells: z.array(
    z.object({
      entity: z.string(),
      metric: z.string(),
      period: z.string(),
      reason: z.string(),
    }),
  ),
  appliedMethodologies: z.array(z.string()).default([]),
  notes: z.array(z.string()).default([]),
});

function buildUserMessage(input: AnalyticalTableInput, job: JobRequest): string {
  return [
    `## Analytical Table — runbook execution`,
    ``,
    `## JobRequest`,
    `analysisId: ${job.analysisId}`,
    `question:   ${job.question}`,
    `period:     ${job.period}`,
    `entities:   ${job.entities.map(e => e.id).join(', ')}`,
    `targetMetrics:`,
    ...job.targetMetrics.map(m => `  - ${m.key}${m.unit ? ` (target unit ${m.unit})` : ''}`),
    ``,
    `## Pillar 1 resolved records (${input.records.length})`,
    JSON.stringify(input.records, null, 2),
    ``,
    `## What to do`,
    `Run your 10-step runbook:`,
    `  1-2. Verify Pillar 1 lineage on each record. Decide table structure: rows=entities, cols=metrics, layers=periods.`,
    `  3-5. For unit / period / entity normalization, query find_methodologies(type="normalization_rule") first; if the`,
    `       library has a relevant entry use it, otherwise apply a sensible structural pass and note the methodology gap.`,
    `  6.   For every (entity × metric × period) requested by the JobRequest that has NO matching record, add an entry to`,
    `       missingCells with a clear reason. Never invent values.`,
    `  7.   Stamp each cell's sourceLineage from the upstream record's sourceUrl.`,
    `  8-9. Validate the table; assign per-cell confidence (inherit upstream, lower if you made a normalization choice).`,
    `  10.  Return the table.`,
    ``,
    `## Output`,
    `Return ONLY a JSON object — no prose, no markdown fence — in this exact shape:`,
    `{`,
    `  "entities":  string[],`,
    `  "metrics":   string[],`,
    `  "periods":   string[],`,
    `  "cells": [`,
    `    {`,
    `      "entity":         string,`,
    `      "metric":         string,`,
    `      "period":         string,`,
    `      "value":          number | null,`,
    `      "unit":           string,`,
    `      "derivations":    string[],          // e.g. ["unit:identity:USD", "period:fy=2024 fp=FY"]`,
    `      "sourceLineage":  string[],          // source URLs from upstream record(s)`,
    `      "confidence":     number,            // 0-1`,
    `      "flags":          string[]`,
    `    }`,
    `  ],`,
    `  "missingCells": [`,
    `    { "entity": string, "metric": string, "period": string, "reason": string }`,
    `  ],`,
    `  "appliedMethodologies": string[],      // methodology_id values you invoked from the library`,
    `  "notes":               string[]`,
    `}`,
  ].join('\n');
}

function jsonResponseFromText(text: string): unknown {
  const cleaned = text.replace(/^```(?:json)?\s*/i, '').replace(/```\s*$/i, '').trim();
  try { return JSON.parse(cleaned); } catch { /* fall through */ }
  const m = cleaned.match(/\{[\s\S]*\}/);
  if (!m) return null;
  try { return JSON.parse(m[0]); } catch { return null; }
}

function summarize(name: string, ok: boolean, result: unknown): string {
  if (!ok) return 'error';
  if (name === 'find_methodologies' && Array.isArray(result)) {
    return `${result.length} match(es): ${result.slice(0, 5).map((r: any) => r.methodology_id).join(', ')}`;
  }
  if (name === 'get_methodology' && result && typeof result === 'object') {
    const r = result as { methodology_id?: string; name?: string };
    return `${r.methodology_id ?? '?'} — ${r.name ?? ''}`;
  }
  return 'ok';
}

export interface BuildTableResult {
  readonly table: AnalyticalTableOutput;
  readonly toolCalls: readonly ToolCallTrace[];
}

export async function buildAnalyticalTable(
  input: AnalyticalTableInput,
  job: JobRequest,
  onToolCall?: (t: ToolCallTrace) => void,
): Promise<LlmResult<BuildTableResult>> {
  if (!client) {
    return {
      ok: false,
      failure: {
        category: 'needs-api-key',
        reason: 'Analytical Table requires the LLM but ANTHROPIC_API_KEY is not configured.',
        hint: 'Set ANTHROPIC_API_KEY and rerun.',
      },
    };
  }

  const system = buildSystemPrompt();
  const messages: MessageParam[] = [{ role: 'user', content: buildUserMessage(input, job) }];
  const toolCalls: ToolCallTrace[] = [];

  let finalText = '';
  for (let iter = 0; iter < MAX_TOOL_ITERATIONS; iter++) {
    let resp;
    try {
      resp = await client.messages.create({
        model: MODEL,
        max_tokens: MAX_TOKENS_PER_TURN,
        system,
        tools: ANTHROPIC_TOOLS,
        messages,
      });
    } catch (err) {
      return {
        ok: false,
        failure: { category: 'sdk-error', reason: err instanceof Error ? err.message : String(err) },
      };
    }
    recordUsage('intelligence.analytical-table', MODEL, resp.usage.input_tokens, resp.usage.output_tokens);

    messages.push({ role: 'assistant', content: resp.content as ContentBlock[] });

    if (resp.stop_reason !== 'tool_use') {
      const textBlock = resp.content.find((b): b is TextBlock => b.type === 'text');
      finalText = textBlock ? textBlock.text : '';
      break;
    }

    const toolUses = resp.content.filter((b): b is ToolUseBlock => b.type === 'tool_use');
    const toolResults: { type: 'tool_result'; tool_use_id: string; content: string; is_error?: boolean }[] = [];
    for (const tu of toolUses) {
      const r = await executeMethodologyTool(tu.name, tu.input);
      const trace: ToolCallTrace = {
        toolName: tu.name,
        input: (tu.input ?? {}) as Record<string, unknown>,
        ok: r.ok,
        resultSummary: summarize(tu.name, r.ok, r.result),
        errorMessage: r.error?.message,
        at: new Date().toISOString(),
      };
      toolCalls.push(trace);
      onToolCall?.(trace);
      toolResults.push({
        type: 'tool_result',
        tool_use_id: tu.id,
        content: r.ok ? JSON.stringify(r.result) : JSON.stringify({ error: r.error }),
        is_error: !r.ok,
      });
    }
    messages.push({ role: 'user', content: toolResults });
  }

  if (!finalText) {
    return {
      ok: false,
      failure: {
        category: 'tool-loop-overrun',
        reason: `Tool-use loop exceeded ${MAX_TOOL_ITERATIONS} iterations without an end_turn.`,
      },
    };
  }

  const json = jsonResponseFromText(finalText);
  const parsed = responseSchema.safeParse(json);
  if (!parsed.success) {
    return {
      ok: false,
      failure: {
        category: 'invalid-response',
        reason: `final response did not match AnalyticalTable schema: ${parsed.error.message}`,
      },
    };
  }
  return { ok: true, value: { table: parsed.data, toolCalls } };
}