BID · Console
Baseline · Intelligence · Decision
scripts/intel-verify.ts 6,759 bytes · typescript
/**
 * intel-verify — free verification of the SME methodology library
 * and the Anthropic tool surface that Intelligence agents use.
 *
 * No Anthropic spend. Loads every YAML under
 * src/intelligence/methodologies/, runs a battery of sample queries
 * against the library, exercises the executeMethodologyTool dispatcher
 * the same way the agents do, and prints structured output for visual
 * verification.
 *
 *   npm run intel:verify
 */

import {
  loadLibrary,
  findMethodologies,
  getMethodology,
  type Methodology,
} from '../src/intelligence/library.js';
import { executeMethodologyTool, METHODOLOGY_TOOLS } from '../src/intelligence/tools.js';

function banner(s: string): void {
  console.log(`\n══════════════════════════════════════════════════════════`);
  console.log(s);
  console.log(`══════════════════════════════════════════════════════════`);
}

function row(label: string, value: unknown): void {
  console.log(`  ${label.padEnd(18)} ${value}`);
}

function summarise(m: Methodology): string {
  return `${m.methodology_id.padEnd(40)} type=${m.type.padEnd(20)} domain=${m.domain.padEnd(14)} status=${m.status.padEnd(10)} (${m.sourceFile})`;
}

async function main(): Promise<void> {
  const t0 = Date.now();

  /* ----------------- 1. Library load ----------------- */
  banner('1. Library load (YAML scan + index)');
  const all = await loadLibrary();
  row('entries loaded:', all.length);
  for (const m of all) console.log(`  - ${summarise(m)}`);

  /* ----------------- 2. Schema view of one entry ----------------- */
  banner('2. Full content of tech_opex_efficiency_banking');
  const tech = await getMethodology('tech_opex_efficiency_banking');
  if (!tech) {
    console.log('  (missing — expected to be present)');
  } else {
    row('name:', tech.name);
    row('type:', tech.type);
    row('domain:', tech.domain);
    row('agent:', tech.applies_to.agent);
    row('triggers:', tech.applies_to.triggers.length);
    for (const t of tech.applies_to.triggers) console.log(`                       • ${t}`);
    row('definition:', JSON.stringify(tech.definition));
    row('inputs:', JSON.stringify(tech.inputs));
    row('outputs:', JSON.stringify(tech.outputs));
    row('status:', tech.status);
    row('source file:', tech.sourceFile);
  }

  /* ----------------- 3. Sample queries ----------------- */
  banner('3. Sample queries (library.ts API)');
  const queries: { label: string; query: Parameters<typeof findMethodologies>[0] }[] = [
    { label: 'type=metric_definition',                       query: { type: 'metric_definition' } },
    { label: 'type=comparison_method',                       query: { type: 'comparison_method' } },
    { label: 'type=insight_framework',                       query: { type: 'insight_framework' } },
    { label: 'domain=banking',                               query: { domain: 'banking' } },
    { label: 'domain=insurance (no entries yet)',            query: { domain: 'insurance' } },
    { label: 'agent=performance_metrics',                    query: { agent: 'performance_metrics' } },
    { label: 'agent=comparisons_synthesis',                  query: { agent: 'comparisons_synthesis' } },
    { label: 'triggers=["growth","trend"]',                  query: { triggers: ['growth', 'trend'] } },
    { label: 'triggers=["technology"]',                      query: { triggers: ['technology'] } },
    { label: 'triggers=["nonsense"] (should be empty)',      query: { triggers: ['nonsense'] } },
  ];
  for (const q of queries) {
    const hits = await findMethodologies(q.query);
    console.log(`  ${q.label.padEnd(48)} → ${hits.length} hit(s): ${hits.map(h => h.methodology_id).join(', ') || '—'}`);
  }

  /* ----------------- 4. Anthropic tool surface ----------------- */
  banner('4. Anthropic tool descriptors (what agents declare to Claude)');
  for (const t of METHODOLOGY_TOOLS) {
    console.log(`  - ${t.name}`);
    console.log(`      ${t.description.replace(/\s+/g, ' ').slice(0, 110)}…`);
    console.log(`      required: [${t.input_schema.required.join(', ')}]  properties: [${Object.keys(t.input_schema.properties).join(', ')}]`);
  }

  /* ----------------- 5. Execute tools (same path the agents take) ----------------- */
  banner('5. executeMethodologyTool — exercises the dispatcher');

  const callA = await executeMethodologyTool('find_methodologies', { type: 'metric_definition' });
  console.log(`  find_methodologies({type:"metric_definition"}) → ok=${callA.ok}, ${(callA.result as unknown[] | undefined)?.length ?? 0} hit(s)`);

  const callB = await executeMethodologyTool('find_methodologies', { triggers: 'growth, trend' });
  console.log(`  find_methodologies({triggers:"growth, trend"}) → ok=${callB.ok}, ${(callB.result as unknown[] | undefined)?.length ?? 0} hit(s)`);

  const callC = await executeMethodologyTool('get_methodology', { methodology_id: 'peer_benchmark_three_year_growth' });
  const peer = callC.result as Methodology | undefined;
  console.log(`  get_methodology("peer_benchmark_three_year_growth") → ok=${callC.ok} name="${peer?.name ?? ''}"`);

  const callD = await executeMethodologyTool('get_methodology', { methodology_id: 'does_not_exist' });
  console.log(`  get_methodology("does_not_exist") → ok=${callD.ok} error=${callD.error?.category ?? '?'}: ${callD.error?.message ?? ''}`);

  const callE = await executeMethodologyTool('get_methodology', {});
  console.log(`  get_methodology({}) → ok=${callE.ok} error=${callE.error?.category ?? '?'}: ${callE.error?.message ?? ''}`);

  const callF = await executeMethodologyTool('unknown_tool', {});
  console.log(`  unknown_tool({}) → ok=${callF.ok} error=${callF.error?.category ?? '?'}: ${callF.error?.message ?? ''}`);

  /* ----------------- 6. Per-agent reachability ----------------- */
  banner('6. Which methodologies each Intelligence agent can find today');
  const agents = ['analytical_table', 'performance_metrics', 'comparisons_synthesis', 'insight_synthesis'] as const;
  for (const a of agents) {
    const hits = await findMethodologies({ agent: a });
    console.log(`  ${a.padEnd(24)} → ${hits.length} reachable methodology(ies): ${hits.map(h => h.methodology_id).join(', ') || '—'}`);
  }
  console.log(`  (any agent whose row is "—" will escalate every relevant request as a methodology gap)`);

  console.log(`\nDone. Elapsed: ${((Date.now() - t0) / 1000).toFixed(2)}s. $0 spent.`);
}

main().catch(err => {
  console.error('intel-verify failed:', err);
  process.exit(1);
});