Move media_store & logs out of /data. Allow logging to be configured

The goal is to allow these to be on separate partitions
(including remote ones in the future).

Because the `silviof/docker-matrix` image chowns
everything to MATRIX_UID:MATRIX_GID on startup,
we definitely don't want to include `media_store` in it.
If it's on a remote FS, it would cause a slow startup.

Also, adding some safety checks to the "import media store"
task, after passing a wrong path to it on multiple occassions and
wondering what's wrong.

Also, making logging configurable. The default of keeping 10x100MB
log files is likely excessive and people may want to change that.

roles/matrix-server/defaults/main.yml

@@ -24,7 +24,10 @@ matrix_postgres_db_name: "homeserver"
 
 matrix_base_data_path: "/matrix"
 matrix_environment_variables_data_path: "{{ matrix_base_data_path }}/environment-variables"
-matrix_synapse_data_path: "{{ matrix_base_data_path }}/synapse"
+matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse"
+matrix_synapse_config_dir_path: "{{ matrix_synapse_base_path }}/config"
+matrix_synapse_run_path: "{{ matrix_synapse_base_path }}/run"
+matrix_synapse_media_store_path: "{{ matrix_synapse_base_path }}/media-store"
 matrix_postgres_data_path: "{{ matrix_base_data_path }}/postgres"
 matrix_nginx_proxy_data_path: "{{ matrix_base_data_path }}/nginx-proxy"
 matrix_nginx_proxy_confd_path: "{{ matrix_nginx_proxy_data_path }}/conf.d"
@@ -46,4 +49,6 @@ matrix_coturn_turn_udp_max_port: 49172
 
 matrix_coturn_turn_external_ip_address: "{{ ansible_host }}"
 
-matrix_max_upload_size_mb: 10
+matrix_max_upload_size_mb: 10
+matrix_max_log_file_size_mb: 100
+matrix_max_log_files_count: 10

roles/matrix-server/tasks/import_media_store.yml

@@ -14,6 +14,22 @@
   fail: msg="File cannot be found on the local machine at {{ local_path_media_store }}"
   when: "not local_path_media_store_stat.stat.exists or not local_path_media_store_stat.stat.isdir"
 
+- name: Check if media store contains local_content
+  stat: path="{{ local_path_media_store }}/local_content"
+  delegate_to: 127.0.0.1
+  become: false
+  register: local_path_media_store_local_content_stat
+
+- name: Check if media store contains remote_content
+  stat: path="{{ local_path_media_store }}/remote_content"
+  delegate_to: 127.0.0.1
+  become: false
+  register: local_path_media_store_remote_content_stat
+
+- name: Fail if media_store directory doesn't look okay (lacking remote and local content)
+  fail: msg="{{ local_path_media_store }} contains neither local_content nor remote_content. It's most likely a mistake and is not a media store directory."
+  when: "not local_path_media_store_local_content_stat.stat.exists and not local_path_media_store_remote_content_stat.stat.exists"
+
 - name: Ensure matrix-synapse is stopped
   service: name=matrix-synapse state=stopped daemon_reload=yes
   register: stopping_result
@@ -21,7 +37,7 @@
 - name: Ensure provided media_store directory is copied to the server
   synchronize:
     src: "{{ local_path_media_store }}/"
-    dest: "{{ matrix_synapse_data_path }}/media_store"
+    dest: "{{ matrix_synapse_media_store_path }}"
     delete: yes
 
 - name: Ensure Matrix Synapse is started (if it previously was)

roles/matrix-server/tasks/import_sqlite_db.yml

@@ -66,7 +66,9 @@
     command: "/usr/local/bin/synapse_port_db_with_patch --sqlite-database /scratchpad/homeserver.db --postgres-config /data/homeserver.yaml"
     user: "{{ matrix_user_uid }}:{{ matrix_user_gid }}"
     volumes:
-      - "{{ matrix_synapse_data_path }}:/data"
+      - "{{ matrix_synapse_config_dir_path }}:/data"
+      - "{{ matrix_synapse_run_path }}:/matrix-run"
+      - "{{ matrix_synapse_media_store_path }}:/matrix-media-store"
       - "{{ matrix_scratchpad_dir }}:/scratchpad"
       - "{{ matrix_scratchpad_dir }}/synapse_port_db_with_patch:/usr/local/bin/synapse_port_db_with_patch"
     links:

roles/matrix-server/tasks/setup_synapse.yml

@@ -1,12 +1,17 @@
 ---
 
-- name: Ensure Matrix Synapse data path exists
+- name: Ensure Matrix Synapse paths exists
   file:
-    path: "{{ matrix_synapse_data_path }}"
+    path: "{{ item }}"
     state: directory
     mode: 0750
     owner: "{{ matrix_user_username }}"
     group: "{{ matrix_user_username }}"
+  with_items:
+    - "{{ matrix_synapse_base_path }}"
+    - "{{ matrix_synapse_config_dir_path }}"
+    - "{{ matrix_synapse_run_path }}"
+    - "{{ matrix_synapse_media_store_path }}"
 
 - name: Ensure Matrix Docker image is pulled
   docker_image:
@@ -14,7 +19,7 @@
 
 - name: Check if a Matrix Synapse configuration exists
   stat:
-    path: "{{ matrix_synapse_data_path }}/homeserver.yaml"
+    path: "{{ matrix_synapse_config_dir_path }}/homeserver.yaml"
   register: matrix_synapse_config_stat
 
 - name: Generate initial Matrix config
@@ -29,41 +34,44 @@
       REPORT_STATS: "no"
     user: "{{ matrix_user_uid }}:{{ matrix_user_gid }}"
     volumes:
-      - "{{ matrix_synapse_data_path }}:/data"
+      - "{{ matrix_synapse_config_dir_path }}:/data"
   when: "not matrix_synapse_config_stat.stat.exists"
 
-- name: Augment Matrix config (configure SSL fullchain location)
+- name: Ensure self-signed certificates are removed
-  lineinfile: "dest={{ matrix_synapse_data_path }}/homeserver.yaml"
+  file:
-  args:
+    path: "{{ item }}"
-    regexp: "^tls_certificate_path:"
+    state: absent
-    line: 'tls_certificate_path: "/acmetool-certs/live/{{ hostname_matrix }}/fullchain"'
+  with_items:
+    - "{{ matrix_synapse_config_dir_path }}/{{ hostname_matrix }}.tls.crt"
+    - "{{ matrix_synapse_config_dir_path }}/{{ hostname_matrix }}.tls.key"
 
-- name: Augment Matrix config (configure SSL private key location)
+- name: Augment Matrix log config
-  lineinfile: "dest={{ matrix_synapse_data_path }}/homeserver.yaml"
+  lineinfile: "dest={{ matrix_synapse_config_dir_path }}/{{ hostname_matrix }}.log.config"
   args:
-    regexp: "^tls_private_key_path:"
+    regexp: "{{ item.regexp }}"
-    line: 'tls_private_key_path: "/acmetool-certs/live/{{ hostname_matrix }}/privkey"'
+    line: '{{ item.line }}'
+  with_items:
+    - {"regexp": "^    filename:", "line": '    filename: /matrix-run/homeserver.log'}
+    - {"regexp": "^    maxBytes:", "line": '    maxBytes: {{ matrix_max_log_file_size_mb * 1024 * 1024 }}'}
+    - {"regexp": "^    backupCount:", "line": '    backupCount: {{ matrix_max_log_files_count }}'}
 
-- name: Augment Matrix config (configure server name)
+- name: Augment Matrix config
-  lineinfile: "dest={{ matrix_synapse_data_path }}/homeserver.yaml"
+  lineinfile: "dest={{ matrix_synapse_config_dir_path }}/homeserver.yaml"
   args:
-    regexp: "^server_name:"
+    regexp: "{{ item.regexp }}"
-    line: 'server_name: "{{ hostname_identity }}"'
+    line: '{{ item.line }}'
-
+  with_items:
-- name: Augment Matrix config (disable TURN for guests)
+    - {"regexp": "^log_file:", "line": 'log_file: "/matrix-run/homeserver.log"'}
-  lineinfile: "dest={{ matrix_synapse_data_path }}/homeserver.yaml"
+    - {"regexp": "^tls_certificate_path:", "line": 'tls_certificate_path: "/acmetool-certs/live/{{ hostname_matrix }}/fullchain"'}
-  args:
+    - {"regexp": "^tls_private_key_path:", "line": 'tls_private_key_path: "/acmetool-certs/live/{{ hostname_matrix }}/privkey"'}
-    regexp: "^turn_allow_guests:"
+    - {"regexp": "^server_name:", "line": 'server_name: "{{ hostname_identity }}"'}
-    line: 'turn_allow_guests: False'
+    - {"regexp": "^turn_allow_guests:", "line": 'turn_allow_guests: False'}
-
+    - {"regexp": "^url_preview_enabled:", "line": 'url_preview_enabled: True'}
-- name: Augment Matrix config (enable URL previews)
+    - {"regexp": "^max_upload_size:", "line": 'max_upload_size: "{{ matrix_max_upload_size_mb }}M"'}
-  lineinfile: "dest={{ matrix_synapse_data_path }}/homeserver.yaml"
+    - {"regexp": "^media_store_path:", "line": 'media_store_path: "/matrix-media-store"'}
-  args:
-    regexp: "^url_preview_enabled:"
-    line: 'url_preview_enabled: True'
 
 - name: Augment Matrix config (specify URL previews blacklist)
-  lineinfile: "dest={{ matrix_synapse_data_path }}/homeserver.yaml"
+  lineinfile: "dest={{ matrix_synapse_config_dir_path }}/homeserver.yaml"
   args:
     regexp: "^url_preview_ip_range_blacklist:"
     line: 'url_preview_ip_range_blacklist: ["127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "100.64.0.0/10", "169.254.0.0/16"]'
@@ -72,27 +80,27 @@
 # We only wish to do this for the 8008 port and not for the 8448 port
 # (2nd instance of `x_forwarded` found in the config)
 - name: Augment Matrix config (mark 8008 plain traffic as forwarded)
-  replace: "dest={{ matrix_synapse_data_path }}/homeserver.yaml"
+  replace: "dest={{ matrix_synapse_config_dir_path }}/homeserver.yaml"
   args:
     regexp: "8008((?:.|\n)*)x_forwarded(.*)"
     replace: '8008\g<1>x_forwarded: true'
 
 - name: Augment Matrix config (change database from SQLite to Postgres)
   lineinfile:
-    dest: "{{ matrix_synapse_data_path }}/homeserver.yaml"
+    dest: "{{ matrix_synapse_config_dir_path }}/homeserver.yaml"
     regexp: '(.*)name: "sqlite3"'
     line: '\1name: "psycopg2"'
     backrefs: yes
 
 - name: Augment Matrix config (add the Postgres connection parameters)
   lineinfile:
-    dest: "{{ matrix_synapse_data_path }}/homeserver.yaml"
+    dest: "{{ matrix_synapse_config_dir_path }}/homeserver.yaml"
     regexp: '(.*)database: "(.*)homeserver.db"'
     line: '\1user: "{{ matrix_postgres_connection_username }}"\n\1password: "{{ matrix_postgres_connection_password }}"\n\1database: "homeserver"\n\1host: "postgres"\n\1cp_min: 5\n\1cp_max: 10'
     backrefs: yes
 
 - name: Augment Matrix config (configure Coturn)
-  lineinfile: "dest={{ matrix_synapse_data_path }}/turnserver.conf"
+  lineinfile: "dest={{ matrix_synapse_config_dir_path }}/turnserver.conf"
   args:
     regexp: "^{{ item.variable }}="
     line: '{{ item.variable }}={{ item.value }}'
@@ -101,12 +109,6 @@
     - {'variable': 'max-port', 'value': "{{ matrix_coturn_turn_udp_max_port }}"}
     - {'variable': 'external-ip', 'value': "{{ matrix_coturn_turn_external_ip_address }}"}
 
-- name: Augment Matrix config (set max upload size)
-  lineinfile: "dest={{ matrix_synapse_data_path }}/homeserver.yaml"
-  args:
-    regexp: "^max_upload_size:"
-    line: 'max_upload_size: "{{ matrix_max_upload_size_mb }}M"'
-
 - name: Allow access to Matrix ports in firewalld
   firewalld:
     port: "{{ item }}"

roles/matrix-server/templates/systemd/matrix-synapse.service.j2

@@ -16,7 +16,9 @@ ExecStart=/usr/bin/docker run --rm --name matrix-synapse \
 			-p 3478:3478 \
 			-p 3478:3478/udp \
 			-p {{ matrix_coturn_turn_udp_min_port }}-{{ matrix_coturn_turn_udp_max_port }}:{{ matrix_coturn_turn_udp_min_port }}-{{ matrix_coturn_turn_udp_max_port }}/udp \
-			-v {{ matrix_synapse_data_path }}:/data \
+			-v {{ matrix_synapse_config_dir_path }}:/data \
+			-v {{ matrix_synapse_run_path }}:/matrix-run \
+			-v {{ matrix_synapse_media_store_path }}:/matrix-media-store \
 			-v {{ ssl_certs_path }}:/acmetool-certs \
 			{{ docker_matrix_image }}
 ExecStop=-/usr/bin/docker kill matrix-synapse