LLM 路由与降级策略：不要把所有请求交给一个模型

// 坏做法：所有任务都走同一个模型
async function handleCustomerMessage(message: string) {
  const response = await openai.chat.completions.create({
    model: 'gpt-4',
    messages: [{ role: 'user', content: message }],
  })
  return response.choices[0].message.content
}

// 好做法：按任务类型路由到不同模型
type TaskType = 'faq' | 'summary' | 'reasoning' | 'extraction'
 
// 第一步：用轻量分类器判断任务类型
async function classifyTask(message: string): Promise<TaskType> {
  const response = await openai.chat.completions.create({
    model: 'gpt-4o-mini', // 分类任务用小模型
    messages: [
      {
        role: 'system',
        content: '将用户消息分类为 faq/summary/reasoning/extraction 之一，只返回类别名',
      },
      { role: 'user', content: message },
    ],
    max_tokens: 10,
  })
  return response.choices[0].message.content.trim() as TaskType
}
 
// 第二步：根据任务类型选择模型
const MODEL_MAP: Record<TaskType, string> = {
  faq: 'gpt-4o-mini',          // 简单问答，成本最低
  summary: 'claude-3-5-haiku', // 摘要任务，速度快
  reasoning: 'gpt-4',          // 复杂推理，需要强模型
  extraction: 'gemini-2.0-flash', // 结构化抽取，性价比高
}
 
async function handleCustomerMessage(message: string) {
  const taskType = await classifyTask(message)
  const model = MODEL_MAP[taskType]
 
  const response = await callModel(model, message)
  return response
}

// 坏做法：简单的 if-else 降级，没有区分故障类型
async function callWithFallback(prompt: string) {
  try {
    return await openai.chat.completions.create({
      model: 'gpt-4',
      messages: [{ role: 'user', content: prompt }],
    })
  } catch (error) {
    // 不管什么错误都切换到 Anthropic，太粗暴了
    return await anthropic.messages.create({
      model: 'claude-sonnet-4-20250514',
      messages: [{ role: 'user', content: prompt }],
    })
  }
}

// 好做法：带重试、退避和熔断的降级链
interface RetryConfig {
  maxAttempts: number
  baseDelayMs: number
  maxDelayMs: number
}
 
// 故障分类：不同故障类型对应不同处理策略
type FailureType =
  | 'retryable'      // 5xx、超时、网络错误 → 重试
  | 'rate_limited'   // 429 → 尊重 Retry-After 头部
  | 'content_filter' // 内容安全拒绝 → 不重试，走审核流程
  | 'fatal'          // 参数错误等 → 直接报错
 
function classifyFailure(status: number): FailureType {
  if (status === 429) return 'rate_limited'
  if (status >= 400 && status < 500) return 'content_filter'
  if (status >= 500) return 'retryable'
  return 'fatal'
}
 
// 指数退避 + 抖动：避免重试风暴
async function sleepWithJitter(attempt: number, base: number, cap: number) {
  const delay = Math.min(cap, base * Math.pow(2, attempt))
  const jitter = delay * (0.5 + Math.random() * 0.5)
  await new Promise(resolve => setTimeout(resolve, jitter))
}
 
// 熔断器：防止持续请求已不可用的供应商
class CircuitBreaker {
  private state: 'closed' | 'open' | 'half_open' = 'closed'
  private failureCount = 0
  private lastFailureTime = 0
 
  constructor(
    private threshold: number = 5,
    private cooldownMs: number = 30_000,
  ) {}
 
  canExecute(): boolean {
    if (this.state === 'closed') return true
    if (this.state === 'open') {
      if (Date.now() - this.lastFailureTime > this.cooldownMs) {
        this.state = 'half_open'
        return true // 放一个请求试探
      }
      return false
    }
    return true // half_open 状态允许一个请求
  }
 
  recordSuccess() {
    this.failureCount = 0
    this.state = 'closed'
  }
 
  recordFailure() {
    this.failureCount++
    this.lastFailureTime = Date.now()
    if (this.failureCount >= this.threshold) {
      this.state = 'open'
    }
  }
}

# 声明式降级链配置：把策略从代码中分离出来
rules:
  - id: primary-chat
    type: priority-based-routing
    when:
      task: chat
    targets:
      - target: openai/gpt-4
        priority: 0
        retry:
          max_attempts: 3
          on_status_codes: [429, 500, 502, 503]
        timeout_ms: 15000
        circuit_breaker:
          threshold: 5
          cooldown_ms: 30000
      - target: anthropic/claude-sonnet-4-20250514
        priority: 1
        retry:
          max_attempts: 2
          on_status_codes: [429, 500, 502, 503]
        timeout_ms: 20000
      - target: self-hosted/qwen-2.5-72b
        priority: 2
        timeout_ms: 30000

// 坏做法：直接拿主模型的 prompt 给备选模型用
const SHARED_PROMPT = `你是一个客服助手。
请用以下格式回答：
1. 先确认问题
2. 给出解决方案
3. 如果没有解决方案，说"I cannot help with this"
...`
 
// 好做法：为每个模型维护独立的 prompt 适配层
interface ModelPromptAdapter {
  modelId: string
  adaptPrompt(basePrompt: string): string
  validateOutput(output: string): boolean
}
 
const adapters: Record<string, ModelPromptAdapter> = {
  'gpt-4': {
    modelId: 'gpt-4',
    adaptPrompt: (base) => base, // GPT-4 用原始 prompt
    validateOutput: (output) => !output.includes('I cannot'),
  },
  'claude-sonnet-4-20250514': {
    modelId: 'claude-sonnet-4-20250514',
    adaptPrompt: (base) =>
      base
        .replace('I cannot help with this', '我需要将您的问题转交人工客服')
        .replace(/请严格按以下/g, '请尽量参考以下'), // Claude 对强制指令更敏感
    validateOutput: (output) =>
      output.length > 0 && !output.includes('I cannot'),
  },
}

// 方案 A：服务端缓冲后统一发送（丢失流式的延迟优势）
async function bufferedFallback(request: Request) {
  const fullResponse = await callPrimary(request) // 等待完整响应
  if (!fullResponse) {
    const fallbackResponse = await callSecondary(request) // 切换
    return fallbackResponse // 一次性返回
  }
  return fullResponse
}
 
// 方案 B：先探测再流式（推荐的折中方案）
async function probeThenStream(request: Request) {
  // 用非流式请求先探测供应商是否可用
  const probe = await callPrimary({
    ...request,
    max_tokens: 5, // 只生成几个 token 探测
    stream: false,
  })
 
  if (probe.ok) {
    // 探测成功，放心地开始流式传输
    return streamFromPrimary(request)
  }
  // 探测失败，直接用备选模型流式
  return streamFromSecondary(request)
}
 
// 方案 C：接受中途重启（需要 UX 配合）
async function streamWithRestart(request: Request, emitter: SSEEmitter) {
  const stream = await streamFromPrimary(request)
  let tokensSent = 0
 
  for await (const chunk of stream) {
    if (chunk.error && tokensSent > 0) {
      // 已经发送了部分 tokens，无法透明切换
      emitter.send({ type: 'restart', model: 'secondary' })
      const fallbackStream = await streamFromSecondary(request)
      for await (const fb of fallbackStream) {
        emitter.send(fb)
      }
      return
    }
    emitter.send(chunk)
    tokensSent++
  }
}