附录 G：Agent 评估 — 你怎么知道你的 Agent 够好了

// → 可靠性测试伪代码
function measureReliability(agent, task, runs = 5): ReliabilityScore {
  const results = []
  for (let i = 0; i < runs; i++) {
    results.push(agent.run(task))
  }

  const successCount = results.filter(r => r.success).length
  const consistency = successCount / runs  // 0-1

  // 即使全部成功，也要看结果是否相同
  if (consistency === 1.0) {
    const allSameResult = results.every(r =>
      deepEqual(r.output, results[0].output)
    )
    return { consistency: 1.0, deterministic: allSameResult }
  }

  return { consistency, deterministic: false }
}

// → 工具选择评估示例
const toolChoiceTests = [
  {
    prompt: "读取 src/auth.ts 的内容",
    expectedTool: "Read",
    expectedArgs: { file_path: "src/auth.ts" }
  },
  {
    prompt: "运行所有单元测试",
    expectedTool: "Bash",
    expectedArgs: { command: "npm test" }
  },
  {
    prompt: "搜索 all files containing 'TODO'",
    expectedTool: "Grep",
    expectedArgs: { pattern: "TODO" }
  },
]

// 评估
function evaluateToolChoice(agent, tests): ToolChoiceScore {
  let correct = 0
  for (const test of tests) {
    const call = agent.decideTool(test.prompt)
    if (call.name === test.expectedTool) correct++
  }
  return { accuracy: correct / tests.length, total: tests.length }
}

function evaluateArgs(
  expected: Record<string, unknown>,
  actual: Record<string, unknown>
): number {
  const keys = Object.keys(expected)
  let score = 0

  for (const key of keys) {
    if (actual[key] === expected[key]) {
      score += 1
    } else if (typeof expected[key] === "string" && typeof actual[key] === "string") {
      // 字符串部分匹配（如文件路径）
      if (actual[key].includes(expected[key])) score += 0.5
    }
  }

  return score / keys.length  // 0-1
}

// → eval-runner.ts — 最简 Agent 评估器
import { AgentFramework } from "./src/framework"
import { readFileSync } from "fs"

interface EvalTask {
  id: string
  description: string
  verification: { type: "test"; command: string } | { type: "output"; expected: string }
}

interface EvalResult {
  taskId: string
  success: boolean
  turns: number
  tokens: { input: number; output: number; total: number }
  cost: number
  durationMs: number
  error?: string
}

async function evaluateAgent(
  framework: AgentFramework,
  tasks: EvalTask[],
): Promise<EvalResult[]> {
  const results: EvalResult[] = []

  for (const task of tasks) {
    const start = Date.now()
    const agent = framework.createAgent()

    try {
      for await (const event of agent.run(task.description)) {
        if (event.type === "response") {
          const success = await verify(task.verification)
          results.push({
            taskId: task.id,
            success,
            turns: agent.turnCount,
            tokens: agent.totalTokens,
            cost: agent.sessionCost,
            durationMs: Date.now() - start,
          })
        }
      }
    } catch (err) {
      results.push({
        taskId: task.id,
        success: false,
        turns: agent.turnCount,
        tokens: agent.totalTokens,
        cost: agent.sessionCost,
        durationMs: Date.now() - start,
        error: err.message,
      })
    }
  }

  return results
}

async function verify(verification: EvalTask["verification"]): Promise<boolean> {
  if (verification.type === "test") {
    const { execSync } = await import("child_process")
    try {
      execSync(verification.command, { timeout: 30_000 })
      return true
    } catch { return false }
  }
  return false
}

// → 带预算的评估
async function evaluateWithBudget(
  framework: AgentFramework,
  tasks: EvalTask[],
  budget: number,  // 最大金额（美元）
): Promise<EvalResult[]> {
  let spent = 0
  const results: EvalResult[] = []

  for (const task of tasks) {
    if (spent > budget) {
      console.warn(`评估预算用尽 ($${spent.toFixed(2)} / $${budget})，已评估 ${results.length}/${tasks.length} 个任务`)
      break
    }

    const agent = framework.createAgent({ maxCostPerSession: budget - spent })
    const result = await runSingleTask(agent, task)
    spent += result.cost
    results.push(result)
  }

  return results
}