diff --git a/docs/configuring-playbook-prometheus-grafana.md b/docs/configuring-playbook-prometheus-grafana.md index b7f3caae4..4edc2a81f 100644 --- a/docs/configuring-playbook-prometheus-grafana.md +++ b/docs/configuring-playbook-prometheus-grafana.md @@ -58,6 +58,38 @@ Name | Description `matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_key`|Set this to a password to use for HTTP Basic Auth for protecting `https://matrix.DOMAIN/_synapse/metrics` (the username is always `prometheus` - it's not configurable) `matrix_server_fqn_grafana`|Use this variable to override the domain at which the Grafana web user-interface is at (defaults to `stats.DOMAIN`) +### Collecting worker metrics to an external Prometheus server + +If you are using workers (`matrix_synapse_workers_enabled`) and have enabled `matrix_nginx_proxy_proxy_synapse_metrics` as described above, the playbook will also automatically proxy the all worker threads's metrics to `https://matrix.DOMAIN/_synapse-worker-TYPE-ID/metrics`, where `TYPE` corresponds to the type and `ID` to the instanceId of a worker as exemplified in `matrix_synapse_workers_enabled_list`. + +The playbook also generates an exemplary prometheus.yml config file (`matrix_base_data_path/external_prometheus.yml.template`) with all the correct paths which you can copy to your Prometheus server and adapt to your needs, especially edit the specified `password_file` path and contents and path to your `synapse-v2.rules`. +It will look a bit like this: +```yaml +scrape_configs: + - job_name: 'synapse' + metrics_path: /_synapse/metrics + scheme: https + basic_auth: + username: prometheus + password_file: /etc/prometheus/password.pwd + static_configs: + - targets: ['matrix.DOMAIN:443'] + labels: + job: "master" + index: 1 + - job_name: 'synapse-generic_worker-1' + metrics_path: /_synapse-worker-generic_worker-18111/metrics + scheme: https + basic_auth: + username: prometheus + password_file: /etc/prometheus/password.pwd + static_configs: + - targets: ['matrix.DOMAIN:443'] + labels: + job: "generic_worker" + index: 18111 +``` + ### Collecting system and Postgres metrics to an external Prometheus server (advanced) When you normally enable the Prometheus and Grafana via the playbook, it will also show general system (via node-exporter) and Postgres (via postgres-exporter) stats. If you are instead collecting your metrics to an external Prometheus server, you can follow this advanced configuration example to also export these stats. diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 4905a222e..6d56307ec 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1339,6 +1339,8 @@ matrix_nginx_proxy_synapse_media_repository_locations: "{{matrix_synapse_workers matrix_nginx_proxy_synapse_user_dir_locations: "{{ matrix_synapse_workers_user_dir_endpoints|default([]) }}" matrix_nginx_proxy_synapse_frontend_proxy_locations: "{{ matrix_synapse_workers_frontend_proxy_endpoints|default([]) }}" +matrix_nginx_proxy_proxy_synapse_workers_enabled_list: "{{ matrix_synapse_workers_enabled_list }}" + matrix_nginx_proxy_systemd_wanted_services_list: | {{ (['matrix-synapse.service'] if matrix_synapse_enabled else []) diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 88ebd7cd1..3bd273cf5 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -182,6 +182,7 @@ matrix_nginx_proxy_proxy_matrix_identity_api_addr_sans_container: "127.0.0.1:809 # Controls whether proxying for metrics (`/_synapse/metrics`) should be done (on the matrix domain) matrix_nginx_proxy_proxy_synapse_metrics: false +matrix_nginx_proxy_synapse_workers_enabled_list: [] matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled: false # The following value will be written verbatim to the htpasswd file that stores the password for nginx to check against and needs to be encoded appropriately. # Read the manpage at `man 1 htpasswd` to learn more, then encrypt your password, and paste the encrypted value here. @@ -226,7 +227,7 @@ matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_prefix_regexes: | + (['/_synapse/admin'] if matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_admin_api_enabled else []) + - (['/_synapse/metrics'] if matrix_nginx_proxy_proxy_synapse_metrics else []) + (['/_synapse.*/metrics'] if matrix_nginx_proxy_proxy_synapse_metrics else []) }} # Specifies where requests for the root URI (`/`) on the `matrix.` domain should be redirected. diff --git a/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml b/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml index 149fadab7..e577491b8 100644 --- a/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml +++ b/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml @@ -38,6 +38,15 @@ mode: 0400 when: "matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled|bool and matrix_nginx_proxy_proxy_synapse_metrics|bool" +- name: Generate sample prometheus.yml for external scraping + template: + src: "{{ role_path }}/templates/prometheus/external_prometheus.yml.example.j2" + dest: "{{ matrix_base_data_path }}/external_prometheus.yml.example" + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + mode: 0644 + when: matrix_nginx_proxy_proxy_synapse_metrics|bool + - name: Ensure Matrix nginx-proxy configured (generic) template: src: "{{ role_path }}/templates/nginx/conf.d/nginx-http.conf.j2" @@ -270,3 +279,9 @@ path: "{{ matrix_nginx_proxy_data_path }}/matrix-synapse-metrics-htpasswd" state: absent when: "not matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled|bool or not matrix_nginx_proxy_proxy_synapse_metrics|bool" + +- name: Ensure sample prometheus.yml for external scraping is deleted + file: + path: "{{ matrix_base_data_path }}/external_prometheus.yml.example" + state: absent + when: "not matrix_nginx_proxy_proxy_synapse_metrics|bool" diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index db111090c..6f569327d 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -153,6 +153,24 @@ server { } {% endif %} + {% if matrix_nginx_proxy_enabled and matrix_nginx_proxy_proxy_synapse_metrics %} + {% for worker in matrix_nginx_proxy_proxy_synapse_workers_enabled_list %} + {% if worker.metrics_port != 0 %} + location /_synapse-worker-{{ worker.type }}-{{ worker.instanceId }}/metrics { + resolver 127.0.0.11 valid=5s; + set $backend "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.metrics_port }}"; + proxy_pass http://$backend/_synapse/metrics; + proxy_set_header Host $host; + + {% if matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled %} + auth_basic "protected"; + auth_basic_user_file /nginx-data/matrix-synapse-metrics-htpasswd; + {% endif %} + } + {% endif %} + {% endfor %} + {% endif %} + {# Everything else just goes to the API server ##} location / { {% if matrix_nginx_proxy_enabled %} diff --git a/roles/matrix-nginx-proxy/templates/prometheus/external_prometheus.yml.example.j2 b/roles/matrix-nginx-proxy/templates/prometheus/external_prometheus.yml.example.j2 new file mode 100644 index 000000000..01a39ffa5 --- /dev/null +++ b/roles/matrix-nginx-proxy/templates/prometheus/external_prometheus.yml.example.j2 @@ -0,0 +1,40 @@ +global: + scrape_interval: 5s + + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'synapse-{{ matrix_domain }}' + +rule_files: + - /etc/prometheus/synapse-v2.rules + +scrape_configs: + - job_name: 'synapse' + metrics_path: /_synapse/metrics + scheme: {{ 'https' if matrix_nginx_proxy_https_enabled else 'http' }} +{% if matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled %} + basic_auth: + username: prometheus + password_file: /path/to/your/passwordfile.pwd +{% endif %} + static_configs: + - targets: ['{{ matrix_server_fqn_matrix }}:{{ matrix_nginx_proxy_container_https_host_bind_port if matrix_nginx_proxy_https_enabled else matrix_nginx_proxy_container_http_host_bind_port }}'] + labels: + job: "master" + index: 1 +{% for worker in matrix_nginx_proxy_proxy_synapse_workers_enabled_list %} + - job_name: 'synapse-{{ worker.type }}-{{ worker.instanceId }}' + metrics_path: /_synapse-worker-{{ worker.type }}-{{ worker.instanceId }}/metrics + scheme: {{ 'https' if matrix_nginx_proxy_https_enabled else 'http' }} +{% if matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled %} + basic_auth: + username: prometheus + password_file: /path/to/your/passwordfile.pwd +{% endif %} + static_configs: + - targets: ['{{ matrix_server_fqn_matrix }}:{{ matrix_nginx_proxy_container_https_host_bind_port if matrix_nginx_proxy_https_enabled else matrix_nginx_proxy_container_http_host_bind_port }}'] + labels: + job: "{{ worker.type }}" + index: {{ worker.instanceId }} +{% endfor %}