From 9c87bba7111009c09d37a3e0fd5b128f39fa9985 Mon Sep 17 00:00:00 2001 From: Ilayda Dastan <43909097+ilaydadastan@users.noreply.github.com> Date: Wed, 3 Jul 2024 19:00:18 +0300 Subject: [PATCH] feat(jitsi-meet): added grafana loki and otel integration for log analysis (#1844) --- .gitignore | 3 + grafana.yml | 13 ++ log-analyser.yml | 21 +++ log-analyser/README.md | 79 ++++++++ log-analyser/grafana-dashboards/jicofo.json | 177 ++++++++++++++++++ .../grafana-dashboards/jitsi-all.json | 136 ++++++++++++++ log-analyser/grafana-dashboards/jvb.json | 177 ++++++++++++++++++ .../datasources/datasource_loki.yml | 9 + log-analyser/loki/conf/loki-config.yaml | 51 +++++ log-analyser/otel-collector-config.yaml | 53 ++++++ 10 files changed, 719 insertions(+) create mode 100644 grafana.yml create mode 100644 log-analyser.yml create mode 100644 log-analyser/README.md create mode 100644 log-analyser/grafana-dashboards/jicofo.json create mode 100644 log-analyser/grafana-dashboards/jitsi-all.json create mode 100644 log-analyser/grafana-dashboards/jvb.json create mode 100644 log-analyser/grafana-provisioning/datasources/datasource_loki.yml create mode 100644 log-analyser/loki/conf/loki-config.yaml create mode 100644 log-analyser/otel-collector-config.yaml diff --git a/.gitignore b/.gitignore index 9c7dffa..8996a5d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ *.swp .env* docker-compose.override.yml +log-analyser/grafana +**/.DS_Store +**/.idea diff --git a/grafana.yml b/grafana.yml new file mode 100644 index 0000000..37d143c --- /dev/null +++ b/grafana.yml @@ -0,0 +1,13 @@ +version: '3.5' + +services: + # Grafana: used for visualization of metrics and log data through customizable dashboards. + grafana: + image: grafana/grafana:10.2.0 + environment: + - GF_ANALYTICS_REPORTING_ENABLED=false + volumes: + - ./log-analyser/grafana:/var/lib/grafana + - ./log-analyser/grafana-provisioning/:/etc/grafana/provisioning/ + ports: + - "3000:3000" diff --git a/log-analyser.yml b/log-analyser.yml new file mode 100644 index 0000000..a4fc8ee --- /dev/null +++ b/log-analyser.yml @@ -0,0 +1,21 @@ +version: '3.5' + +services: + # Log Analyser: used for setting up a log analysis system for visualization, log collection and log processing. + + loki: + container_name: loki + image: grafana/loki:3.0.0 + command: -config.file=/conf/loki-config.yaml + volumes: + - ./log-analyser/loki/data:/data + - ./log-analyser/loki/conf:/conf + ports: + - "3100:3100" + + otel-collector: + container_name: otel + image: otel/opentelemetry-collector-contrib + volumes: + - ./log-analyser/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml + - ./log-analyser/jitsi-logs/:/tmp/jitsi-logs/ diff --git a/log-analyser/README.md b/log-analyser/README.md new file mode 100644 index 0000000..eb47993 --- /dev/null +++ b/log-analyser/README.md @@ -0,0 +1,79 @@ +# JITSI MEET LOG ANALYSER - Grafana Loki and OpenTelemetry Integration + +Welcome to the Grafana Loki and OpenTelemetry integration project! This repository provides a simple and effective setup for log management and analysis using Docker, Grafana Loki, and OpenTelemetry. +Currently this is an in-progress GSoC Summer of Code project and so the instructions may change before being finalized. Please treat all this as alpha code. + +## Overview + +This project demonstrates how to configure and use Grafana Loki with OpenTelemetry to collect, parse, and visualize log data. It includes: + +- A Docker Compose setup (`log-analyser.yml`) for Loki and OpenTelemetry Collector. +- A Docker Compose setup (`grafana.yml`) for Grafana. +- Configuration files for log parsing and exporting. +- Instructions to set up and access Grafana with Loki as a data source. +- +## Getting Started + +### Prerequisites + +- Docker +- Docker Compose + +### Setup + +1. **Clone the repository:** + + ```bash + git clone https://github.com/jitsi/docker-jitsi-meet.git + ``` + +### Log Analyser + +1. **Add your log files:** + + Place your log file in the `log-analyser/jitsi-logs` directory. Update the `otel-collector-config.yaml` file with the correct file path to start ingesting the logs. This setup allows OpenTelemetry to read logs from the file and forward them to Loki. + +2. **Update the otel-collector-config.yaml file:** + + Update the file path to point to your log file for ingestion by OpenTelemetry. + +3. **Start the Docker containers:** + + ```bash + docker-compose -f docker-compose.yml -f log-analyser.yml up -d + ``` + +### Grafana + +1. **Start the Docker container:** + + ```bash + docker-compose -f docker-compose.yml -f grafana.yml up -d + ``` + +2. **Access Grafana:** + + Open your web browser and navigate to [http://localhost:3000](http://localhost:3000). + +3. **Log in to Grafana:** + + Use the default credentials: + + ``` + Username: admin + Password: admin + ``` + +### Dashboard Setup + +The dashboard setups are available as JSON files in the `log-analyser/grafana-dashboards` directory. You can import these JSON files into Grafana to use the pre-configured dashboards. In the future, we plan to automate this import process. + + +## Usage + +- **Log Parsing and Visualization:** After setting up, you can use Grafana to explore and visualize your logs. Check our dashboards and panels to monitor log data effectively. + + +## Acknowledgements + +Thanks for checking out this project! If you have any questions or need further assistance, don't hesitate to reach out. diff --git a/log-analyser/grafana-dashboards/jicofo.json b/log-analyser/grafana-dashboards/jicofo.json new file mode 100644 index 0000000..2024180 --- /dev/null +++ b/log-analyser/grafana-dashboards/jicofo.json @@ -0,0 +1,177 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "b8130a28-4867-4668-917d-539c93852857" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false, + "values": [] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "b8130a28-4867-4668-917d-539c93852857" + }, + "editorMode": "code", + "expr": "sum by (attributes_level) (\n rate({exporter=\"OTLP\"} | json|attributes_log_file_name=\"jicofo.log\"| line_format \"{{.log}}\" | logfmt | pattern \"[<_>] <_level>: <_>\"[5m])\n)", + "legendFormat": "Level: {{attributes_level}}", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log Levels Pie Chart", + "type": "piechart" + }, + { + "datasource": { + "type": "loki", + "uid": "b8130a28-4867-4668-917d-539c93852857" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "displayMode": "lcd", + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "valueMode": "color" + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "b8130a28-4867-4668-917d-539c93852857" + }, + "editorMode": "code", + "expr": "sum by (attributes_level, attributes_log_file_name) (\n rate({exporter=\"OTLP\"} | json|attributes_log_file_name=\"jicofo.log\"| line_format \"{{.attributes_message}}\" | logfmt | pattern \"[<_>] #: <_>\"[5m]))", + "legendFormat": "Level: {{attributes_level}}", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log Levels Bar Chart", + "transformations": [], + "type": "bargauge" + } + ], + "refresh": "", + "schemaVersion": 38, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Jicofo Logs", + "uid": "f2dcfe84-3c27-4b1d-8583-bc2c97a8d22d", + "version": 5, + "weekStart": "" +} diff --git a/log-analyser/grafana-dashboards/jitsi-all.json b/log-analyser/grafana-dashboards/jitsi-all.json new file mode 100644 index 0000000..922ad2e --- /dev/null +++ b/log-analyser/grafana-dashboards/jitsi-all.json @@ -0,0 +1,136 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "b8130a28-4867-4668-917d-539c93852857" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "barRadius": 0, + "barWidth": 1, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "b8130a28-4867-4668-917d-539c93852857" + }, + "editorMode": "code", + "expr": "sum by (attributes_log_file_name) (\n rate({exporter=\"OTLP\"} | json | line_format \"{{.attributes_message}}\" | logfmt | pattern \"[<_>] <_level>: <_>\"[5m]))", + "legendFormat": "{{attributes_log_file_name}}", + "queryType": "range", + "refId": "A" + } + ], + "title": "Jicofo and JVB Log Counts", + "type": "barchart" + } + ], + "refresh": "", + "schemaVersion": 38, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Jitsi All Components", + "uid": "b75d666d-4537-45e2-94a1-2783f9362b65", + "version": 1, + "weekStart": "" +} diff --git a/log-analyser/grafana-dashboards/jvb.json b/log-analyser/grafana-dashboards/jvb.json new file mode 100644 index 0000000..9a2be5c --- /dev/null +++ b/log-analyser/grafana-dashboards/jvb.json @@ -0,0 +1,177 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "b8130a28-4867-4668-917d-539c93852857" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "displayLabels": [ + "percent", + "name" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "b8130a28-4867-4668-917d-539c93852857" + }, + "editorMode": "code", + "expr": "sum by (attributes_level) (\n rate({exporter=\"OTLP\"} | json|attributes_log_file_name=\"jvb.log\"| line_format \"{{.log}}\" | logfmt | pattern \"[<_>] <_level>: <_>\"[5m])\n)", + "legendFormat": "Level: {{attributes_level}}", + "queryType": "range", + "refId": "A" + } + ], + "title": "JVB Log Levels Pie Chart", + "type": "piechart" + }, + { + "datasource": { + "type": "loki", + "uid": "b8130a28-4867-4668-917d-539c93852857" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "displayMode": "lcd", + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "valueMode": "color" + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "b8130a28-4867-4668-917d-539c93852857" + }, + "editorMode": "code", + "expr": "sum by (attributes_level, attributes_log_file_name) (\n rate({exporter=\"OTLP\"} | json|attributes_log_file_name=\"jvb.log\"| line_format \"{{.attributes_message}}\" | logfmt | pattern \"[<_>] #: <_>\"[5m]))", + "legendFormat": "Level: {{attributes_level}}", + "queryType": "range", + "refId": "A" + } + ], + "title": "JVB Log Levels Bar Chart", + "type": "bargauge" + } + ], + "refresh": "", + "schemaVersion": 38, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "JVB Dashboard", + "uid": "d53a9efb-ca3b-4f47-af3a-9638de8a35fa", + "version": 1, + "weekStart": "" +} diff --git a/log-analyser/grafana-provisioning/datasources/datasource_loki.yml b/log-analyser/grafana-provisioning/datasources/datasource_loki.yml new file mode 100644 index 0000000..6c8883d --- /dev/null +++ b/log-analyser/grafana-provisioning/datasources/datasource_loki.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: + - name: Loki + isDefault: true + type: loki + access: proxy + url: http://loki:3100 + editable: true \ No newline at end of file diff --git a/log-analyser/loki/conf/loki-config.yaml b/log-analyser/loki/conf/loki-config.yaml new file mode 100644 index 0000000..0d42a30 --- /dev/null +++ b/log-analyser/loki/conf/loki-config.yaml @@ -0,0 +1,51 @@ +limits_config: + allow_structured_metadata: true + +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + + +# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration +# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ +# +# Statistics help us better understand how Loki is used, and they show us performance +# levels for most users. This helps us prioritize features and documentation. +# For more information on what's sent, look at +# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go +# Refer to the buildReport method to see what goes into a report. +# +# If you would like to disable reporting, uncomment the following lines: +#analytics: +# reporting_enabled: false diff --git a/log-analyser/otel-collector-config.yaml b/log-analyser/otel-collector-config.yaml new file mode 100644 index 0000000..d9f1636 --- /dev/null +++ b/log-analyser/otel-collector-config.yaml @@ -0,0 +1,53 @@ +receivers: + otlp: + protocols: + http: + grpc: + endpoint: 0.0.0.0:4317 + filelog/jicofo: + include: ['/tmp/jitsi-logs/jicofo.log'] + operators: + - type: regex_parser + regex: "^(?PJicofo) (?P\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (?P\\w+): \\[(?P\\d+)\\] (?P[\\w\\.]+)#(?P\\d+): (?P[\\S\\s]*)$" + filelog/jvb: + include: ['/tmp/jitsi-logs/jvb.log'] + operators: + - type: regex_parser + regex: "^(?PJVB) (?P\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (?P\\w+): \\[(?P\\d+)\\] (?P[\\w\\.]+)#(?P\\d+): (?P[\\S\\s]*)$" + filelog/prosody: + include: ['/tmp/jitsi-logs/prosody.log'] + operators: + - type: regex_parser + regex: "^(?P\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})\\s+(?P\\S+)\\s+(?P\\w+)\\s+(?P.+)$" + filelog/jitsi-web: + include: ['/tmp/jitsi-logs/jitsi-web.log'] + operators: + - type: regex_parser + regex: "^(?P\\[\\w+-\\w+\\.\\w+\\]\\s+(\\w+\\s+)*\\w+:?\\s*.*$)" +processors: + batch: + +exporters: + logging: + loglevel: debug + loki: + endpoint: "http://loki:3100/loki/api/v1/push" + +service: + pipelines: + logs/jicofo: + receivers: [otlp, filelog/jicofo] + processors: [batch] + exporters: [loki] + logs/jvb: + receivers: [otlp, filelog/jvb] + processors: [batch] + exporters: [loki] + logs/prosody: + receivers: [otlp, filelog/prosody] + processors: [batch] + exporters: [loki] + logs/jitsi-web: + receivers: [otlp, filelog/jitsi-web] + processors: [batch] + exporters: [loki]