Update from live: sidekiq performance, prometheus mappings

docker-compose.yml

@@ -14,6 +14,7 @@ services:
     env_file: .env.production
     environment:
       - 'POSTGRES_HOST_AUTH_METHOD=trust'
+    command: postgres -c 'max_connections=250'
 
   mastodon_redis:
     container_name: mastodon_redis
@@ -100,13 +101,36 @@ services:
       - mastodon_db
       - mastodon_redis
 
+  # https://thomas-leister.de/en/scaling-up-mastodon/
   mastodon_sidekiq:
     container_name: mastodon_sidekiq
     build: .
     image: ghcr.io/mastodon/mastodon:v4.2.0
     restart: always
     env_file: .env.production
-    command: bundle exec sidekiq
+    environment:
+      - DB_POOL=100
+    command: bundle exec sidekiq -c 100
+    depends_on:
+      - mastodon_db
+      - mastodon_redis
+    networks:
+      - mastodon
+      - nginx
+    volumes:
+      - ./public/system:/mastodon/public/system
+    healthcheck:
+      test: ['CMD-SHELL', "ps aux | grep '[s]idekiq\ 6' || false"]
+
+  mastodon_sidekiq_2:
+    container_name: mastodon_sidekiq_2
+    build: .
+    image: ghcr.io/mastodon/mastodon:v4.1.5
+    restart: always
+    env_file: .env.production
+    environment:
+      - DB_POOL=100
+    command: bundle exec sidekiq -c 100
     depends_on:
       - mastodon_db
       - mastodon_redis
@@ -141,6 +165,10 @@ services:
     restart: always
     ports:
       - 0.0.0.0:9102:9102
+    command:
+      "--statsd.mapping-config=/statsd-mapping.yaml"
+    volumes:
+      - ./statsd-mapping.yaml:/statsd-mapping.yaml
     networks:
       - mastodon
 

statsd-mapping.yaml

@@ -0,0 +1,96 @@
+## Prometheus Statsd Exporter mapping for Mastodon 4.0+
+##
+## Version 1.0, November 2022
+##
+## Documentation: https://ipng.ch/s/articles/2022/11/27/mastodon-3.html
+
+mappings:
+  ## Web collector
+  - match: Mastodon\.production\.web\.(.+)\.(.+)\.(.+)\.status\.(.+)
+    match_type: regex
+    name: "mastodon_controller_status"
+    labels:
+      controller: $1
+      action: $2
+      format: $3
+      status: $4
+      mastodon: "web"
+  - match: Mastodon\.production\.web\.(.+)\.(.+)\.(.+)\.db_time
+    match_type: regex
+    name: "mastodon_controller_db_time"
+    labels:
+      controller: $1
+      action: $2
+      format: $3
+      mastodon: "web"
+  - match: Mastodon\.production\.web\.(.+)\.(.+)\.(.+)\.view_time
+    match_type: regex
+    name: "mastodon_controller_view_time"
+    labels:
+      controller: $1
+      action: $2
+      format: $3
+      mastodon: "web"
+  - match: Mastodon\.production\.web\.(.+)\.(.+)\.(.+)\.total_duration
+    match_type: regex
+    name: "mastodon_controller_duration"
+    labels:
+      controller: $1
+      action: $2
+      format: $3
+      mastodon: "web"
+
+  ## Database collector
+  - match: Mastodon\.production\.db\.tables\.(.+)\.queries\.(.+)\.duration
+    match_type: regex
+    name: "mastodon_db_operation"
+    labels:
+      table: "$1"
+      operation: "$2"
+      mastodon: "db"
+
+  ## Cache collector
+  - match: Mastodon\.production\.cache\.(.+)\.duration
+    match_type: regex
+    name: "mastodon_cache_duration"
+    labels:
+      operation: "$1"
+      mastodon: "cache"
+
+  ## Sidekiq collector
+  - match: Mastodon\.production\.sidekiq\.(.+)\.processing_time
+    match_type: regex
+    name: "mastodon_sidekiq_worker_processing_time"
+    labels:
+      worker: "$1"
+      mastodon: "sidekiq"
+  - match: Mastodon\.production\.sidekiq\.(.+)\.success
+    match_type: regex
+    name: "mastodon_sidekiq_worker_success_total"
+    labels:
+      worker: "$1"
+      mastodon: "sidekiq"
+  - match: Mastodon\.production\.sidekiq\.(.+)\.failure
+    match_type: regex
+    name: "mastodon_sidekiq_worker_failure_total"
+    labels:
+      worker: "$1"
+      mastodon: "sidekiq"
+  - match: Mastodon\.production\.sidekiq\.queues\.(.+)\.enqueued
+    match_type: regex
+    name: "mastodon_sidekiq_queue_enqueued"
+    labels:
+      queue: "$1"
+      mastodon: "sidekiq"
+  - match: Mastodon\.production\.sidekiq\.queues\.(.+)\.latency
+    match_type: regex
+    name: "mastodon_sidekiq_queue_latency"
+    labels:
+      queue: "$1"
+      mastodon: "sidekiq"
+  - match: Mastodon\.production\.sidekiq\.(.+)
+    match_type: regex
+    name: "mastodon_sidekiq_$1"
+    labels:
+      mastodon: "sidekiq"
+