可观测性驱动架构：先设计怎么排障

// ❌ 坏做法：没有 Trace 上下文，没有结构化字段
app.post('/payment/callback', async (req, res) => {
  const startTime = Date.now()
  try {
    const order = await db.orders.findById(req.body.orderId)
    await paymentService.verify(req.body)
    order.status = 'paid'
    await order.save()
    console.log('payment success')
    res.json({ ok: true })
  } catch (err) {
    console.log('payment failed', err.message)
    res.status(500).json({ ok: false })
  }
  // 只记录了总耗时，没有分步耗时
  console.log('duration:', Date.now() - startTime)
})

// ✅ 好做法：结构化日志 + Trace 上下文 + 分步耗时
import { trace } from '@opentelemetry/api'
 
app.post('/payment/callback', async (req, res) => {
  const tracer = trace.getTracer('payment-callback')
  const span = tracer.startSpan('payment.callback')
 
  try {
    // 分步记录 Span，每步都有独立耗时
    const order = await tracer.startActiveSpan('db.findOrder', async (childSpan) => {
      try {
        return await db.orders.findById(req.body.orderId)
      } finally {
        childSpan.end()
      }
    })
 
    await tracer.startActiveSpan('payment.verify', async (childSpan) => {
      await paymentService.verify(req.body)
      childSpan.end()
    })
 
    order.status = 'paid'
    await db.orders.save(order)
 
    // 结构化日志携带 traceId，可在可观测性平台关联
    logger.info('payment.success', {
      traceId: span.spanContext().traceId,
      orderId: order.id,
      amount: order.amount,
    })
 
    span.setStatus({ code: SpanStatusCode.OK })
    res.json({ ok: true })
  } catch (err) {
    // 记录错误细节到 Span，而不只是日志
    span.recordException(err)
    span.setStatus({ code: SpanStatusCode.ERROR, message: err.message })
    logger.error('payment.failed', {
      traceId: span.spanContext().traceId,
      error: err.message,
      orderId: req.body.orderId,
    })
    res.status(500).json({ ok: false })
  } finally {
    span.end()
  }
})

// ❌ 坏做法：每个服务日志格式不同，无关联 ID
// 订单服务
console.log(JSON.stringify({ event: 'order.created', id: '12345' }))
 
// 库存服务
console.log(`Stock deducted for order 12345 at ${new Date().toISOString()}`)
 
// 支付服务
log4j.info('Payment processed: 12345')
 
// 通知服务 - 直接 stdout，没有日志收集
console.log('notification sent')

// ✅ 好做法：统一结构化日志 + 全链路 correlationId
// 公共日志格式定义
interface StructuredLog {
  level: 'info' | 'warn' | 'error'
  service: string
  correlationId: string
  event: string
  timestamp: string
  data?: Record<string, unknown>
}
 
// 订单服务
logger.info({
  service: 'order-service',
  correlationId: orderId,
  event: 'order.created',
  timestamp: new Date().toISOString(),
  data: { userId: 'u-001', amount: 299 },
})
 
// 库存服务 - 从消息 header 取 correlationId
logger.info({
  service: 'inventory-service',
  correlationId: message.headers.correlationId,
  event: 'inventory.deducted',
  timestamp: new Date().toISOString(),
  data: { skuId: 'sku-100', quantity: 1 },
})
 
// 支付服务
logger.info({
  service: 'payment-service',
  correlationId: message.headers.correlationId,
  event: 'payment.completed',
  timestamp: new Date().toISOString(),
  data: { method: 'alipay', transactionId: 'txn-789' },
})
 
// 通知服务
logger.info({
  service: 'notification-service',
  correlationId: message.headers.correlationId,
  event: 'notification.sent',
  timestamp: new Date().toISOString(),
  data: { channel: 'sms', template: 'order-confirmed' },
})

// ❌ 坏做法：只监控基础设施指标
// 没有监控缓存命中率，问题发现太晚
 
// ✅ 好做法：缓存指标 + 熔断 + 降级
import { metrics } from '@opentelemetry/api'
 
const cacheHitCounter = metrics
  .getMeter('cache')
  .createCounter('cache.hits', { description: 'Cache hit count' })
const cacheMissCounter = metrics
  .getMeter('cache')
  .createCounter('cache.misses', { description: 'Cache miss count' })
 
async function getProductDetail(productId: string) {
  const cacheKey = `product:${productId}`
 
  // 1. 先查缓存
  const cached = await redis.get(cacheKey)
  if (cached) {
    cacheHitCounter.add(1, { resource: 'product-detail' })
    return JSON.parse(cached)
  }
 
  // 2. 缓存未命中，记录指标
  cacheMissCounter.add(1, { resource: 'product-detail' })
 
  // 3. 防止缓存击穿：互斥锁只让一个请求穿透
  const lockKey = `lock:${cacheKey}`
  const acquired = await redis.set(lockKey, '1', 'EX', 10, 'NX')
 
  if (!acquired) {
    // 没拿到锁，等一小段时间后重试缓存
    await new Promise((r) => setTimeout(r, 100))
    const retryCached = await redis.get(cacheKey)
    if (retryCached) return JSON.parse(retryCached)
 
    // 降级：返回简化版本，不查数据库
    return getSimplifiedProduct(productId)
  }
 
  try {
    const product = await db.products.findById(productId)
    await redis.setex(cacheKey, 300, JSON.stringify(product))
    return product
  } finally {
    await redis.del(lockKey)
  }
}

# 缓存未命中率告警
- alert: HighCacheMissRate
  expr: |
    rate(cache_misses_total{resource="product-detail"}[5m])
    / rate(cache_hits_total{resource="product-detail"}[5m])
    > 0.2
  for: 1m
  labels:
    severity: warning
  annotations:
    summary: "商品详情缓存未命中率超过 20%"
    action: "检查缓存是否大面积过期，考虑预热或增加 TTL"

# 可用性 SLI：非 5xx 请求占比
sum(rate(http_requests_total{service="api", status!~"5.."}[30d]))
/
sum(rate(http_requests_total{service="api"}[30d]))
 
# P99 延迟 SLI
histogram_quantile(0.99,
  sum(rate(http_request_duration_seconds_bucket{service="api"}[30d])) by (le)
)

# 1 小时窗口：短窗口高消耗速率，说明正在快速消耗预算
- alert: BurnRateHighShort
  expr: |
    sum(rate(http_requests_total{status=~"5.."}[1h]))
    / sum(rate(http_requests_total[1h]))
    > 14.4 * (1 - 0.999)
  for: 2m
 
# 24 小时窗口：长窗口持续消耗，说明问题不是瞬时的
- alert: BurnRateHighLong
  expr: |
    sum(rate(http_requests_total{status=~"5.."}[24h]))
    / sum(rate(http_requests_total[24h]))
    > 3 * (1 - 0.999)
  for: 5m

interface BusinessEvent {
  // 事件名称，使用 domain.action 格式
  event: string
  // 关联 ID，用于跨服务追踪
  correlationId: string
  // 事件时间
  timestamp: string
  // 业务数据
  data: {
    userId?: string
    orderId?: string
    amount?: number
    // 其他业务字段
  }
  // 事件结果
  outcome: 'success' | 'failure'
  // 失败原因（仅 failure 时）
  failureReason?: string
}
 
// 使用示例
function emitOrderEvent(order: Order, outcome: 'success' | 'failure', reason?: string) {
  logger.info({
    event: 'order.completed',
    correlationId: order.id,
    timestamp: new Date().toISOString(),
    data: {
      userId: order.userId,
      orderId: order.id,
      amount: order.totalAmount,
    },
    outcome,
    failureReason: reason,
  })
}

# 告警路由示例
routes:
  # P0：直接影响用户，需要立即响应
  - match:
      severity: critical
    receiver: 'pagerduty-oncall'
    repeat_interval: 5m
 
  # P1：影响部分用户，需要 30 分钟内响应
  - match:
      severity: warning
    receiver: 'slack-engineering'
    repeat_interval: 30m
 
  # P2：潜在风险，工作时间处理
  - match:
      severity: info
    receiver: 'slack-engineering'
    group_wait: 1h

// 初始化 OpenTelemetry SDK
import { NodeSDK } from '@opentelemetry/sdk-node'
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node'
 
const sdk = new NodeSDK({
  traceExporter: new OTLPTraceExporter({
    url: 'http://otel-collector:4318/v1/traces',
  }),
  metricReader: new OTLPMetricExporter({
    url: 'http://otel-collector:4318/v1/metrics',
  }),
  instrumentations: [getNodeAutoInstrumentations()],
})
 
sdk.start()