feat(jitsi-meet): added grafana loki and otel integration for log analysis (#1844)

.gitignore

@@ -1,3 +1,6 @@
 *.swp
 .env*
 docker-compose.override.yml
+log-analyser/grafana
+**/.DS_Store
+**/.idea

grafana.yml

@@ -0,0 +1,13 @@
+version: '3.5'
+
+services:
+  # Grafana: used for visualization of metrics and log data through customizable dashboards.
+  grafana:
+    image: grafana/grafana:10.2.0
+    environment:
+      - GF_ANALYTICS_REPORTING_ENABLED=false
+    volumes:
+      - ./log-analyser/grafana:/var/lib/grafana
+      - ./log-analyser/grafana-provisioning/:/etc/grafana/provisioning/
+    ports:
+      - "3000:3000"

log-analyser.yml

@@ -0,0 +1,21 @@
+version: '3.5'
+
+services:
+  # Log Analyser: used for setting up a log analysis system for visualization, log collection and log processing.
+
+  loki:
+    container_name: loki
+    image: grafana/loki:3.0.0
+    command: -config.file=/conf/loki-config.yaml
+    volumes:
+      - ./log-analyser/loki/data:/data
+      - ./log-analyser/loki/conf:/conf
+    ports:
+      - "3100:3100"
+
+  otel-collector:
+    container_name: otel
+    image: otel/opentelemetry-collector-contrib
+    volumes:
+      - ./log-analyser/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml
+      - ./log-analyser/jitsi-logs/:/tmp/jitsi-logs/

log-analyser/README.md

@@ -0,0 +1,79 @@
+# JITSI MEET LOG ANALYSER - Grafana Loki and OpenTelemetry Integration
+
+Welcome to the Grafana Loki and OpenTelemetry integration project! This repository provides a simple and effective setup for log management and analysis using Docker, Grafana Loki, and OpenTelemetry.
+Currently this is an in-progress GSoC Summer of Code project and so the instructions may change before being finalized.  Please treat all this as alpha code.
+
+## Overview
+
+This project demonstrates how to configure and use Grafana Loki with OpenTelemetry to collect, parse, and visualize log data. It includes:
+
+- A Docker Compose setup (`log-analyser.yml`) for Loki and OpenTelemetry Collector.
+- A Docker Compose setup (`grafana.yml`) for Grafana.
+- Configuration files for log parsing and exporting.
+- Instructions to set up and access Grafana with Loki as a data source.
+- 
+## Getting Started
+
+### Prerequisites
+
+- Docker
+- Docker Compose
+
+### Setup
+
+1. **Clone the repository:**
+
+    ```bash
+    git clone https://github.com/jitsi/docker-jitsi-meet.git
+    ```
+
+### Log Analyser
+
+1. **Add your log files:**
+
+   Place your log file in the `log-analyser/jitsi-logs` directory. Update the `otel-collector-config.yaml` file with the correct file path to start ingesting the logs. This setup allows OpenTelemetry to read logs from the file and forward them to Loki.
+
+2. **Update the otel-collector-config.yaml file:**
+
+   Update the file path to point to your log file for ingestion by OpenTelemetry.
+
+3. **Start the Docker containers:**
+
+   ```bash
+   docker-compose -f docker-compose.yml -f log-analyser.yml up -d
+    ```
+
+### Grafana
+
+1. **Start the Docker container:**
+
+   ```bash
+   docker-compose -f docker-compose.yml -f grafana.yml up -d
+    ```
+
+2. **Access Grafana:**
+
+   Open your web browser and navigate to [http://localhost:3000](http://localhost:3000).
+
+3. **Log in to Grafana:**
+
+   Use the default credentials:
+
+    ```
+    Username: admin
+    Password: admin
+    ```
+
+### Dashboard Setup
+
+The dashboard setups are available as JSON files in the `log-analyser/grafana-dashboards` directory. You can import these JSON files into Grafana to use the pre-configured dashboards. In the future, we plan to automate this import process.
+
+
+## Usage
+
+- **Log Parsing and Visualization:** After setting up, you can use Grafana to explore and visualize your logs. Check our dashboards and panels to monitor log data effectively.
+
+
+## Acknowledgements
+
+Thanks for checking out this project! If you have any questions or need further assistance, don't hesitate to reach out.

log-analyser/grafana-dashboards/jicofo.json

@@ -0,0 +1,177 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 1,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "b8130a28-4867-4668-917d-539c93852857"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            }
+          },
+          "mappings": []
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "displayLabels": [
+          "percent"
+        ],
+        "legend": {
+          "calcs": [],
+          "displayMode": "hidden",
+          "placement": "right",
+          "showLegend": false,
+          "values": []
+        },
+        "pieType": "pie",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "b8130a28-4867-4668-917d-539c93852857"
+          },
+          "editorMode": "code",
+          "expr": "sum by (attributes_level) (\n  rate({exporter=\"OTLP\"} | json|attributes_log_file_name=\"jicofo.log\"| line_format \"{{.log}}\" | logfmt | pattern \"[<_>] <_level>: <_>\"[5m])\n)",
+          "legendFormat": "Level: {{attributes_level}}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Log Levels Pie Chart",
+      "type": "piechart"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "b8130a28-4867-4668-917d-539c93852857"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "continuous-GrYlRd"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green"
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 1,
+      "options": {
+        "displayMode": "lcd",
+        "minVizHeight": 10,
+        "minVizWidth": 0,
+        "namePlacement": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showUnfilled": true,
+        "valueMode": "color"
+      },
+      "pluginVersion": "10.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "b8130a28-4867-4668-917d-539c93852857"
+          },
+          "editorMode": "code",
+          "expr": "sum by (attributes_level, attributes_log_file_name) (\n  rate({exporter=\"OTLP\"} | json|attributes_log_file_name=\"jicofo.log\"| line_format \"{{.attributes_message}}\" | logfmt | pattern \"[<_>] <attributes_level>#<attributes_log_file_name>: <_>\"[5m]))",
+          "legendFormat": "Level: {{attributes_level}}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Log Levels Bar Chart",
+      "transformations": [],
+      "type": "bargauge"
+    }
+  ],
+  "refresh": "",
+  "schemaVersion": 38,
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Jicofo Logs",
+  "uid": "f2dcfe84-3c27-4b1d-8583-bc2c97a8d22d",
+  "version": 5,
+  "weekStart": ""
+}

log-analyser/grafana-dashboards/jitsi-all.json

@@ -0,0 +1,136 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 3,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "b8130a28-4867-4668-917d-539c93852857"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "fillOpacity": 80,
+            "gradientMode": "opacity",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineWidth": 1,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "barRadius": 0,
+        "barWidth": 1,
+        "fullHighlight": false,
+        "groupWidth": 0.7,
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "orientation": "auto",
+        "showValue": "auto",
+        "stacking": "none",
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        },
+        "xTickLabelRotation": 0,
+        "xTickLabelSpacing": 0
+      },
+      "pluginVersion": "10.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "b8130a28-4867-4668-917d-539c93852857"
+          },
+          "editorMode": "code",
+          "expr": "sum by (attributes_log_file_name) (\n  rate({exporter=\"OTLP\"} | json | line_format \"{{.attributes_message}}\" | logfmt | pattern \"[<_>] <_level>: <_>\"[5m]))",
+          "legendFormat": "{{attributes_log_file_name}}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Jicofo and JVB Log Counts",
+      "type": "barchart"
+    }
+  ],
+  "refresh": "",
+  "schemaVersion": 38,
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Jitsi All Components",
+  "uid": "b75d666d-4537-45e2-94a1-2783f9362b65",
+  "version": 1,
+  "weekStart": ""
+}

log-analyser/grafana-dashboards/jvb.json

@@ -0,0 +1,177 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 2,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "b8130a28-4867-4668-917d-539c93852857"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            }
+          },
+          "mappings": []
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "displayLabels": [
+          "percent",
+          "name"
+        ],
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "pieType": "pie",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "b8130a28-4867-4668-917d-539c93852857"
+          },
+          "editorMode": "code",
+          "expr": "sum by (attributes_level) (\n  rate({exporter=\"OTLP\"} | json|attributes_log_file_name=\"jvb.log\"| line_format \"{{.log}}\" | logfmt | pattern \"[<_>] <_level>: <_>\"[5m])\n)",
+          "legendFormat": "Level: {{attributes_level}}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "JVB Log Levels Pie Chart",
+      "type": "piechart"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "b8130a28-4867-4668-917d-539c93852857"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "continuous-GrYlRd"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 1,
+      "options": {
+        "displayMode": "lcd",
+        "minVizHeight": 10,
+        "minVizWidth": 0,
+        "namePlacement": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showUnfilled": true,
+        "valueMode": "color"
+      },
+      "pluginVersion": "10.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "b8130a28-4867-4668-917d-539c93852857"
+          },
+          "editorMode": "code",
+          "expr": "sum by (attributes_level, attributes_log_file_name) (\n  rate({exporter=\"OTLP\"} | json|attributes_log_file_name=\"jvb.log\"| line_format \"{{.attributes_message}}\" | logfmt | pattern \"[<_>] <attributes_level>#<attributes_log_file_name>: <_>\"[5m]))",
+          "legendFormat": "Level: {{attributes_level}}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "JVB Log Levels Bar Chart",
+      "type": "bargauge"
+    }
+  ],
+  "refresh": "",
+  "schemaVersion": 38,
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "JVB Dashboard",
+  "uid": "d53a9efb-ca3b-4f47-af3a-9638de8a35fa",
+  "version": 1,
+  "weekStart": ""
+}

log-analyser/grafana-provisioning/datasources/datasource_loki.yml

@@ -0,0 +1,9 @@
+apiVersion: 1
+
+datasources:
+  - name: Loki
+    isDefault: true
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    editable: true

log-analyser/loki/conf/loki-config.yaml

@@ -0,0 +1,51 @@
+limits_config:
+  allow_structured_metadata: true
+
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+
+# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
+# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
+#
+# Statistics help us better understand how Loki is used, and they show us performance
+# levels for most users. This helps us prioritize features and documentation.
+# For more information on what's sent, look at
+# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go
+# Refer to the buildReport method to see what goes into a report.
+#
+# If you would like to disable reporting, uncomment the following lines:
+#analytics:
+#  reporting_enabled: false

log-analyser/otel-collector-config.yaml

@@ -0,0 +1,53 @@
+receivers:
+  otlp:
+    protocols:
+      http:
+      grpc:
+        endpoint: 0.0.0.0:4317
+  filelog/jicofo:
+    include: ['/tmp/jitsi-logs/jicofo.log']
+    operators:
+      - type: regex_parser
+        regex: "^(?P<app>Jicofo) (?P<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (?P<level>\\w+): \\[(?P<pid>\\d+)\\] (?P<codefile>[\\w\\.]+)#(?P<codeline>\\d+): (?P<message>[\\S\\s]*)$"
+  filelog/jvb:
+    include: ['/tmp/jitsi-logs/jvb.log']
+    operators:
+      - type: regex_parser
+        regex: "^(?P<app>JVB) (?P<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (?P<level>\\w+): \\[(?P<pid>\\d+)\\] (?P<codefile>[\\w\\.]+)#(?P<codeline>\\d+): (?P<message>[\\S\\s]*)$"
+  filelog/prosody:
+    include: ['/tmp/jitsi-logs/prosody.log']
+    operators:
+      - type: regex_parser
+        regex: "^(?P<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})\\s+(?P<service>\\S+)\\s+(?P<level>\\w+)\\s+(?P<message>.+)$"
+  filelog/jitsi-web:
+    include: ['/tmp/jitsi-logs/jitsi-web.log']
+    operators:
+      - type: regex_parser
+        regex: "^(?P<message>\\[\\w+-\\w+\\.\\w+\\]\\s+(\\w+\\s+)*\\w+:?\\s*.*$)"
+processors:
+  batch:
+
+exporters:
+  logging:
+    loglevel: debug
+  loki:
+    endpoint: "http://loki:3100/loki/api/v1/push"
+
+service:
+  pipelines:
+    logs/jicofo:
+      receivers: [otlp, filelog/jicofo]
+      processors: [batch]
+      exporters: [loki]
+    logs/jvb:
+      receivers: [otlp, filelog/jvb]
+      processors: [batch]
+      exporters: [loki]
+    logs/prosody:
+      receivers: [otlp, filelog/prosody]
+      processors: [batch]
+      exporters: [loki]
+    logs/jitsi-web:
+      receivers: [otlp, filelog/jitsi-web]
+      processors: [batch]
+      exporters: [loki]