Designing for Observability from Day One

// Spring Boot with Micrometer — auto-instruments HTTP, DB, JVM
// application.yml
management:
  metrics:
    export:
      prometheus:
        enabled: true
  endpoints:
    web:
      exposure:
        include: prometheus, health, info
 
# Custom business metrics
@Service
public class OrderService {
    
    private final Counter ordersCreated;
    private final Timer orderProcessingTime;
    
    public OrderService(MeterRegistry registry) {
        this.ordersCreated = Counter.builder("orders.created")
            .description("Total orders created")
            .tag("channel", "web")
            .register(registry);
        
        this.orderProcessingTime = Timer.builder("orders.processing.time")
            .description("Time to process an order end-to-end")
            .percentiles(0.5, 0.95, 0.99)  // Expose p50, p95, p99
            .register(registry);
    }
    
    public Order createOrder(OrderRequest request) {
        return orderProcessingTime.record(() -> {
            Order order = processOrder(request);
            ordersCreated.increment();
            return order;
        });
    }
}

# Convention: {domain}_{entity}_{action}_{unit}
 
# Good
http_server_requests_total{status="200", path="/api/orders"}
http_server_request_duration_seconds_bucket{le="0.1"}
orders_created_total{channel="web", region="eu-west"}
db_connection_pool_active_connections{pool="main"}
 
# Bad
myService_thing1_count
req_dur
custom_metric_xyz

// Bad — unstructured, unsearchable
log.error("Failed to process order " + orderId + " for user " + userId + 
          " after " + duration + "ms: " + exception.getMessage());
 
// Log line in Elasticsearch: 
// "Failed to process order ord-123 for user usr-456 after 1423ms: Timeout"
// → You cannot filter by orderId without string parsing

// Good — structured, every field is searchable
log.error("Order processing failed",
    kv("orderId", orderId),
    kv("userId", userId),
    kv("durationMs", duration),
    kv("errorType", "TIMEOUT"),
    kv("traceId", MDC.get("traceId"))
);
 
// Log line in Elasticsearch:
// { "message": "Order processing failed", "orderId": "ord-123", 
//   "userId": "usr-456", "durationMs": 1423, "traceId": "abc123" }
// → You can filter by orderId, traceId, errorType, etc.

// Filter to set traceId on every incoming request
@Component
public class TraceContextFilter extends OncePerRequestFilter {
    
    @Override
    protected void doFilterInternal(HttpServletRequest request, 
                                    HttpServletResponse response,
                                    FilterChain chain) throws IOException, ServletException {
        // Get traceId from upstream (propagated via W3C traceparent header)
        String traceId = extractTraceId(request);
        if (traceId == null) {
            traceId = UUID.randomUUID().toString().replace("-", "");
        }
        
        MDC.put("traceId", traceId);
        MDC.put("userId", extractUserId(request));
        response.setHeader("X-Trace-Id", traceId);  // Return to client
        
        try {
            chain.doFilter(request, response);
        } finally {
            MDC.clear();
        }
    }
}

<!-- Add the OpenTelemetry Java agent to your build -->
<dependency>
    <groupId>io.opentelemetry.instrumentation</groupId>
    <artifactId>opentelemetry-spring-boot-starter</artifactId>
    <version>2.10.0</version>
</dependency>

# application.yml — configure export
otel:
  service:
    name: order-service
  exporter:
    otlp:
      endpoint: http://tempo:4318   # or Jaeger, Honeycomb, Datadog
  traces:
    sampler:
      probability: 0.1              # 10% sampling in production

// Custom span for a business-critical operation
@Autowired
private Tracer tracer;
 
public PaymentResult processPayment(PaymentRequest request) {
    Span span = tracer.spanBuilder("process-payment")
        .setAttribute("payment.amount", request.getAmount())
        .setAttribute("payment.currency", request.getCurrency())
        .setAttribute("payment.provider", request.getProvider())
        .startSpan();
    
    try (Scope scope = span.makeCurrent()) {
        PaymentResult result = paymentGateway.charge(request);
        span.setAttribute("payment.status", result.getStatus());
        return result;
    } catch (Exception e) {
        span.recordException(e);
        span.setStatus(StatusCode.ERROR, e.getMessage());
        throw e;
    } finally {
        span.end();
    }
}

# Prometheus alert rules
groups:
  - name: order-service
    rules:
      - alert: HighErrorRate
        expr: |
          (
            rate(http_server_requests_total{status=~"5..", service="order-service"}[5m])
            /
            rate(http_server_requests_total{service="order-service"}[5m])
          ) > 0.01
        for: 5m
        labels:
          severity: critical
          team: platform
        annotations:
          summary: "High error rate on order-service ({{ $value | humanizePercentage }})"
          description: "Error rate above 1% for 5 minutes. Check traces for error details."
          runbook_url: "https://wiki.internal/runbooks/order-service-high-error-rate"
          dashboard_url: "https://grafana.internal/d/order-service"
 
      - alert: HighP99Latency
        expr: |
          histogram_quantile(0.99, 
            rate(http_server_request_duration_seconds_bucket{
              service="order-service", path="/api/orders"
            }[5m])
          ) > 2.0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "P99 latency above 2s on POST /api/orders"
          description: "99th percentile latency is {{ $value }}s. Check payment service traces."

Observability Checklist
═══════════════════════════════════════════════
Metrics
  ☐ Four golden signals instrumented
  ☐ Business-level counters/gauges for key operations
  ☐ Custom metrics follow naming convention
  ☐ Alert rules written for p99 latency and error rate
 
Logging
  ☐ Structured JSON logging configured
  ☐ Log level policy documented and followed
  ☐ traceId and userId in MDC for all requests
  ☐ No sensitive data (PII, credentials) in logs
  ☐ Meaningful log messages at INFO for business events
 
Tracing
  ☐ OpenTelemetry auto-instrumentation active
  ☐ Custom spans for business-critical operations
  ☐ traceId propagated in all downstream HTTP calls
  ☐ traceId propagated in all message headers
 
Dashboards
  ☐ Service dashboard exists with four golden signals
  ☐ Dashboard linked from service README
  ☐ Alert runbook exists with first-response steps
 
Runbook
  ☐ Top 3 alert scenarios documented
  ☐ Steps to diagnose each scenario
  ☐ Escalation path documented
═══════════════════════════════════════════════