# Prometheus configuration
global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
  - job_name: 'kubernetes-pods'
    kubernetes_sd_configs:
    - role: pod
    relabel_configs:
    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
      action: keep
      regex: true
    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
      action: replace
      target_label: __metrics_path__
      regex: (.+)

// Go application metrics exposition
package main

import (
    "net/http"
    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
    httpRequests = prometheus.NewCounterVec(
        prometheus.CounterOpts{
            Name: "http_requests_total",
            Help: "Total number of HTTP requests",
        },
        []string{"method", "status"},
    )
)

func init() {
    prometheus.MustRegister(httpRequests)
}

func handler(w http.ResponseWriter, r *http.Request) {
    httpRequests.WithLabelValues(r.Method, "200").Inc()
    w.Write([]byte("Hello World"))
}

func main() {
    http.HandleFunc("/", handler)
    http.Handle("/metrics", promhttp.Handler())
    http.ListenAndServe(":8080", nil)
}

# DataDog Agent configuration
api_key: "your-api-key"
site: "datadoghq.com"
logs_enabled: true
process_config:
  enabled: true
apm_config:
  enabled: true

# Custom metrics
init_config:
instances:
  - prometheus_url: http://localhost:8080/metrics
    namespace: "myapp"
    metrics:
      - http_requests_total
      - go_memstats_alloc_bytes

# Python application with DataDog integration
from datadog import initialize, statsd
import time

initialize(
    api_key='your-api-key',
    app_key='your-app-key'
)

# Custom metrics
@statsd.timed('myapp.request.duration')
def process_request():
    statsd.increment('myapp.request.count')
    # Application logic
    time.sleep(0.1)
    statsd.gauge('myapp.queue.size', 42)

// Node.js application with New Relic
require('newrelic');
const express = require('express');
const app = express();

// Custom events and metrics
const newrelic = require('newrelic');

app.get('/api/users', (req, res) => {
  // Custom metric
  newrelic.recordMetric('Custom/API/Users/RequestCount', 1);
  
  // Custom event
  newrelic.recordCustomEvent('UserAPIAccess', {
    userId: req.user.id,
    endpoint: '/api/users',
    responseTime: Date.now() - req.startTime
  });
  
  res.json({ users: [] });
});

# PromQL queries
http_requests_total{job="api-server", status="200"}
rate(http_requests_total[5m])
histogram_quantile(0.95, http_request_duration_seconds_bucket)

# Complex aggregations
sum(rate(http_requests_total[5m])) by (service)
increase(error_count[1h]) > 100

-- DataDog query syntax
avg:system.cpu.user{environment:production} by {host}
sum:myapp.requests.count{status:error}.as_rate()
anomalies(avg:myapp.response_time{service:api}, 'basic', 2)

-- NRQL queries
SELECT average(duration) FROM Transaction WHERE appName = 'MyApp'
SELECT count(*) FROM Transaction FACET name TIMESERIES
SELECT percentile(responseTime, 95) FROM PageView SINCE 1 hour ago

# Alertmanager rules
groups:
- name: api.rules
  rules:
  - alert: HighErrorRate
    expr: rate(http_requests_total{status=~"5.."}[5m]) > 0.1
    for: 5m
    labels:
      severity: critical
    annotations:
      summary: "High error rate detected"
      description: "{{ $labels.job }} has error rate above 10%"

  - alert: HighLatency
    expr: histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) > 0.5
    for: 2m
    labels:
      severity: warning

# Routing configuration
route:
  group_by: ['alertname']
  group_wait: 10s
  group_interval: 10s
  repeat_interval: 1h
  receiver: 'web.hook'

receivers:
- name: 'web.hook'
  slack_configs:
  - api_url: 'https://hooks.slack.com/services/...'
    channel: '#alerts'

{
  "name": "High error rate on API",
  "type": "metric alert",
  "query": "avg(last_5m):sum:myapp.requests.error{service:api}.as_rate() > 0.1",
  "message": "Error rate is above 10% @slack-alerts",
  "tags": ["service:api", "team:backend"],
  "options": {
    "thresholds": {
      "critical": 0.1,
      "warning": 0.05
    },
    "notify_no_data": true,
    "require_full_window": false
  }
}

// New Relic alert via API
const alert = {
  policy: {
    name: "API Performance Policy",
    incident_preference: "PER_CONDITION"
  },
  conditions: [{
    type: "apm_app_metric",
    name: "High Response Time",
    entities: ["application-id"],
    metric: "response_time_web",
    condition_scope: "application",
    terms: [{
      duration: "5",
      operator: "above",
      priority: "critical",
      threshold: "0.5",
      time_function: "all"
    }]
  }]
};

{
  "dashboard": {
    "title": "API Performance Dashboard",
    "panels": [
      {
        "title": "Request Rate",
        "type": "graph",
        "targets": [
          {
            "expr": "sum(rate(http_requests_total[5m])) by (service)",
            "legendFormat": "{{ service }}"
          }
        ]
      },
      {
        "title": "Error Rate",
        "type": "singlestat",
        "targets": [
          {
            "expr": "sum(rate(http_requests_total{status=~\"5..\"}[5m])) / sum(rate(http_requests_total[5m]))"
          }
        ]
      }
    ]
  }
}

{
  "title": "Application Overview",
  "widgets": [
    {
      "definition": {
        "type": "timeseries",
        "requests": [
          {
            "q": "avg:myapp.response_time{service:api}",
            "display_type": "line"
          }
        ],
        "title": "Response Time"
      }
    },
    {
      "definition": {
        "type": "toplist",
        "requests": [
          {
            "q": "top(avg:myapp.errors{*} by {endpoint}, 10, 'mean', 'desc')"
          }
        ]
      }
    }
  ]
}

# Infrastructure costs (annual)
Compute: 3 x c5.xlarge = $3,000
Storage: 1TB SSD = $1,200
Networking: Data transfer = $500
Staff: 0.5 FTE DevOps = $75,000
Total: ~$79,700/year

# DataDog pricing (annual)
Infrastructure Monitoring: 100 hosts × $15 × 12 = $18,000
APM: 100 hosts × $31 × 12 = $37,200
Log Management: 50GB/day × $1.27 × 365 = $23,206
Custom Metrics: 1M/month × $0.05 × 12 = $600
Total: ~$79,006/year

# New Relic pricing (annual)
Platform: $25/100GB × 12 = $300 (first 100GB)
Additional Data: 500GB × $0.25 × 12 = $1,500
Enterprise Features: $750/month × 12 = $9,000
Total: ~$10,800/year

// OpenTelemetry with Prometheus metrics
import (
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/prometheus"
    "go.opentelemetry.io/otel/exporters/jaeger"
)

func initTracing() {
    // Jaeger for traces
    jaegerExporter, _ := jaeger.New(
        jaeger.WithCollectorEndpoint("http://jaeger:14268/api/traces"),
    )
    
    // Prometheus for metrics
    promExporter, _ := prometheus.New()
    
    tracerProvider := trace.NewTracerProvider(
        trace.WithBatcher(jaegerExporter),
    )
    otel.SetTracerProvider(tracerProvider)
}

# Python APM with DataDog
from ddtrace import patch_all, tracer
patch_all()

@tracer.wrap("database.query")
def query_database(query):
    with tracer.trace("db.execute") as span:
        span.set_tag("db.statement", query)
        span.set_tag("service.name", "user-service")
        return execute_query(query)

// Node.js with New Relic
const newrelic = require('newrelic');

async function processOrder(orderId) {
  return newrelic.startBackgroundTransaction('process-order', async () => {
    const span = newrelic.getTransaction();
    span.addAttribute('orderId', orderId);
    
    // Process order logic
    await paymentService.charge(order);
    await inventoryService.reserve(order);
    
    return order;
  });
}

# Prometheus HA with Thanos
version: '3'
services:
  prometheus-1:
    image: prom/prometheus
    command:
      - '--storage.tsdb.path=/prometheus'
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.min-block-duration=2h'
      - '--storage.tsdb.max-block-duration=2h'
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      
  thanos-sidecar:
    image: thanosio/thanos
    command:
      - 'sidecar'
      - '--tsdb.path=/prometheus'
      - '--prometheus.url=http://prometheus-1:9090'
      - '--objstore.config-file=/bucket.yml'

# Hybrid monitoring approach
# Keep Prometheus for infrastructure metrics
# Add DataDog for APM and business metrics
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
data:
  prometheus.yml: |
    global:
      external_labels:
        region: 'us-west-2'
        environment: 'production'
    
    remote_write:
    - url: "https://api.datadoghq.com/api/v1/series"
      basic_auth:
        username: "datadog"
        password: "api-key"