From 95e02b856b2ac1409dc0d5575ee0b7be55105e9a Mon Sep 17 00:00:00 2001 From: Robert Swain Date: Thu, 28 Sep 2017 12:12:47 +0200 Subject: [PATCH 001/310] docker: Initial Dockerfile and docker-compose.yaml --- Dockerfile | 58 ++++++++++++++++++ docker/README.md | 70 ++++++++++++++++++++++ docker/docker-compose.yaml | 39 ++++++++++++ docker/rootfs/etc/service/synapse/finish | 17 ++++++ docker/rootfs/etc/service/synapse/run | 75 ++++++++++++++++++++++++ 5 files changed, 259 insertions(+) create mode 100644 Dockerfile create mode 100644 docker/README.md create mode 100644 docker/docker-compose.yaml create mode 100755 docker/rootfs/etc/service/synapse/finish create mode 100755 docker/rootfs/etc/service/synapse/run diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..9b11a143f6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,58 @@ +# Copyright 2017 Vector Creations Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM phusion/baseimage:0.9.22 + +COPY ./ /synapse/source/ + +RUN apt-get update -y \ + && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + build-essential \ + libffi-dev \ + libjpeg-dev \ + libpq-dev \ + libssl-dev \ + libxslt1-dev \ + python-pip \ + python-setuptools \ + python-virtualenv \ + python2.7-dev \ + sqlite3 \ + && virtualenv -p python2.7 /synapse \ + && . /synapse/bin/activate \ + && pip install --upgrade pip \ + && pip install --upgrade setuptools \ + && pip install --upgrade psycopg2 \ + && cd /synapse/source \ + && pip install --upgrade ./ \ + && cd / \ + && rm -rf /synapse/source \ + && apt-get autoremove -y \ + build-essential \ + libffi-dev \ + libjpeg-dev \ + libpq-dev \ + libssl-dev \ + libxslt1-dev \ + python2.7-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +COPY docker/rootfs/ / + +VOLUME /synapse/config/ +VOLUME /synapse/data/ + +CMD ["/sbin/my_init"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000000..c9e6fd216d --- /dev/null +++ b/docker/README.md @@ -0,0 +1,70 @@ +# Synapse Docker + +## Build + +Build the docker image with the `docker build` command from the root of the synapse repository. + +``` +docker build -t matrix-org/synapse:v0.22.1 . +``` + +The `-t` option sets the image tag. Official images are tagged `matrix-org/synapse:` where `` is the same as the release tag in the synapse git repository. + +## Configure + +Synapse provides a command for generating homeserver configuration files. These are a good starting point for setting up your own deployment. + +The documentation below will refer to a `CONFIG_PATH` shell variable. This is a path to a directory where synapse configuration will be stored. It needs to be mapped into the container as a volume at `/synapse/config/` as can be seen in the example `docker run` command. + +Docker container environment variables: +* `GENERATE_CONFIG` - Set this to any non-empty string, such as `yes`, to trigger generation of configuration files. Existing files in the `CONFIG_PATH` will **not** be overwritten. +* `POSTGRES_DATABASE` - The database name for the synapse postgres database. [default: `synapse`] +* `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `postgres` which is useful when using a container on the same docker network in a compose file where the postgres service is called `postgres`] **NOTE**: `localhost` and `127.0.0.1` refer to the container itself unless running the container with `host` networking. +* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy. +* `POSTGRES_USER` - The user for the synapse postgres database. [default: `postgres`] +* `REPORT_STATS` - Whether to send anonymous usage statistics back to the Matrix project which helps us to get funding! Must be `yes` or `no`. [default: `yes`] +* `SERVER_NAME` - The domain used for the Matrix homeserver. If you intend to run this synapse instance on a public domain, use that domain. [default: `localhost`] + +``` +CONFIG_PATH=/my/magical/config/path/ +mkdir -p ${CONFIG_PATH} +docker run \ + --rm \ + -e GENERATE_CONFIG=yes \ + -e POSTGRES_PASSWORD=MyVerySecretPassword \ + -e REPORT_STATS=yes \ + -e SERVER_NAME=example.com \ + -v ${CONFIG_PATH}:/synapse/config/ \ + matrix-org/synapse:develop +``` + +This will create a temporary container from the image and use the synapse code for generating configuration files and TLS keys and certificates for the specified `SERVER_NAME` domain. The files are written to `CONFIG_PATH`. + +## Run + +**NOTE**: If you are not using postgresql and are using sqlite3 as your database, you will need to make a directory to store the sqlite3 database file in and then mount this volume into the container at `/synapse/data/`. As it is so easy to use postgresql, when using Docker containers, this is not documented to somewhat discourage it. Choose a `POSTGRES_PASSWORD` instead. + +### Docker Compose + +A `docker-compose.yaml` file is included to ease deployment of the basic synapse and postgres setup. Remember to set a `POSTGRES_PASSWORD` when generating your configuration above. You will need it for running the containers in the composition. + +From the `docker/` subdirectory of the synapse repository: +``` +CONFIG_PATH=/my/magical/config/path/ +POSTGRES_PASSWORD=MyVerySecretPassword \ +docker-compose \ + -p synapse \ + up -d +``` + +### Docker + +Note that the following is just a guideline and you may need to add parameters to the docker run command to account for the network situation with your postgres database. + +``` +docker run \ + -d \ + --name synapse \ + -v ${CONFIG_PATH}:/synapse/config/ \ + matrix-org/synapse:v0.22.1 +``` diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 0000000000..ff36081a9b --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,39 @@ +# Copyright 2017 Vector Creations Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '3' + +services: + postgres: + image: postgres:9.6.5-alpine + environment: + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: synapse + expose: + - 5432 + restart: unless-stopped + volumes: + - postgres-data:/var/lib/postgresql/data/ + + synapse: + image: matrix-org/synapse:develop + ports: + - 8008:8008 + - 8448:8448 + restart: unless-stopped + volumes: + - ${CONFIG_PATH}:/synapse/config/ + +volumes: + postgres-data: diff --git a/docker/rootfs/etc/service/synapse/finish b/docker/rootfs/etc/service/synapse/finish new file mode 100755 index 0000000000..2aace581a1 --- /dev/null +++ b/docker/rootfs/etc/service/synapse/finish @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright 2017 Vector Creations Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kill -TERM 1 diff --git a/docker/rootfs/etc/service/synapse/run b/docker/rootfs/etc/service/synapse/run new file mode 100755 index 0000000000..dd797d3ef9 --- /dev/null +++ b/docker/rootfs/etc/service/synapse/run @@ -0,0 +1,75 @@ +#!/bin/bash +# +# Copyright 2017 Vector Creations Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +: ${CONFIG_PATH:="/synapse/config"} +: ${POSTGRES_DATABASE:="synapse"} +: ${POSTGRES_HOST:="postgres"} +: ${POSTGRES_USER:="postgres"} +: ${REPORT_STATS:="yes"} +: ${SERVER_NAME:="localhost"} + +DATABASE_CONFIG_PATH="${CONFIG_PATH}/database.yaml" +HOMESERVER_CONFIG_PATH="${CONFIG_PATH}/homeserver.yaml" +SYNAPSE_COMMAND="python -m synapse.app.homeserver" + +. /synapse/bin/activate +cd /synapse + +if [[ -n "${GENERATE_CONFIG}" ]]; then + ${SYNAPSE_COMMAND} \ + --server-name ${SERVER_NAME} \ + --config-path ${HOMESERVER_CONFIG_PATH} \ + --generate-config \ + --report-stats=${REPORT_STATS} + + if [[ -f "${DATABASE_CONFIG_PATH}" ]]; then + echo "Config file '${DATABASE_CONFIG_PATH}' already exists. Remove it if you want it to be generated." + else + echo "Generating ${DATABASE_CONFIG_PATH}..." + if [[ -n "${POSTGRES_PASSWORD}" ]]; then + (cat > ${DATABASE_CONFIG_PATH}) < ${DATABASE_CONFIG_PATH}) < Date: Fri, 29 Sep 2017 11:40:15 +0200 Subject: [PATCH 002/310] docker: s/matrix-org/matrixdotorg/g --- docker/README.md | 8 ++++---- docker/docker-compose.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/README.md b/docker/README.md index c9e6fd216d..c15517d0e0 100644 --- a/docker/README.md +++ b/docker/README.md @@ -5,10 +5,10 @@ Build the docker image with the `docker build` command from the root of the synapse repository. ``` -docker build -t matrix-org/synapse:v0.22.1 . +docker build -t matrixdotorg/synapse:v0.22.1 . ``` -The `-t` option sets the image tag. Official images are tagged `matrix-org/synapse:` where `` is the same as the release tag in the synapse git repository. +The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:` where `` is the same as the release tag in the synapse git repository. ## Configure @@ -35,7 +35,7 @@ docker run \ -e REPORT_STATS=yes \ -e SERVER_NAME=example.com \ -v ${CONFIG_PATH}:/synapse/config/ \ - matrix-org/synapse:develop + matrixdotorg/synapse:v0.22.1 ``` This will create a temporary container from the image and use the synapse code for generating configuration files and TLS keys and certificates for the specified `SERVER_NAME` domain. The files are written to `CONFIG_PATH`. @@ -66,5 +66,5 @@ docker run \ -d \ --name synapse \ -v ${CONFIG_PATH}:/synapse/config/ \ - matrix-org/synapse:v0.22.1 + matrixdotorg/synapse:v0.22.1 ``` diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index ff36081a9b..73cc29f8fd 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -27,7 +27,7 @@ services: - postgres-data:/var/lib/postgresql/data/ synapse: - image: matrix-org/synapse:develop + image: matrixdotorg/synapse:v0.22.1 ports: - 8008:8008 - 8448:8448 From 431476fbc4ef0c740e33e19ccc73996c2412e4f9 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sat, 3 Feb 2018 20:18:36 +0100 Subject: [PATCH 003/310] Initial commit including a Dockerfile for synapse --- Dockerfile | 16 ++++++++++++++++ contrib/docker/start.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 Dockerfile create mode 100755 contrib/docker/start.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..5f0433004f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM python:2-alpine + +RUN apk add --no-cache --virtual .nacl_deps build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers + +COPY synapse /usr/local/src/synapse +COPY setup.py setup.cfg README.rst synctl /usr/local/src/ + +RUN cd /usr/local/src \ + && pip install --upgrade --process-dependency-links . \ + && rm -rf setup.py setup.cfg synapse + +COPY contrib/docker / + +VOLUME ["/data"] + +ENTRYPOINT ["/start.py"] diff --git a/contrib/docker/start.py b/contrib/docker/start.py new file mode 100755 index 0000000000..4f63ea1ad5 --- /dev/null +++ b/contrib/docker/start.py @@ -0,0 +1,29 @@ +#!/usr/local/bin/python + +import jinja2 +import os +import sys +import socket + +convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) +mode = sys.argv[1] if len(sys.argv) > 1 else None + +if "SYNAPSE_SERVER_NAME" not in os.environ: + print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.") + sys.exit(2) + +params = ["--server-name", os.environ.get("SYNAPSE_SERVER_NAME"), + "--report-stats", os.environ.get("SYNAPSE_REPORT_STATS", "no"), + "--config-path", os.environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")] + +if mode == "generate": + params.append("--generate-config") + +# Parse the configuration file +if not os.path.exists("/compiled"): + os.mkdir("/compiled") +convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml") +convert("/conf/log.config", "/compiled/%s.log.config" % os.environ.get("SYNAPSE_SERVER_NAME")) + +# TODO, replace with a call to synapse.app.homeserver.run() +os.execv("/usr/local/bin/python", ["python", "-m", "synapse.app.homeserver"] + params) From d434ae33875b4c7d5ee04c45e454ce237ce578f8 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sat, 3 Feb 2018 20:30:08 +0100 Subject: [PATCH 004/310] Add template config files for the Docker image --- contrib/docker/conf/homeserver.yaml | 181 ++++++++++++++++++++++++++++ contrib/docker/conf/log.config | 36 ++++++ 2 files changed, 217 insertions(+) create mode 100644 contrib/docker/conf/homeserver.yaml create mode 100644 contrib/docker/conf/log.config diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml new file mode 100644 index 0000000000..851c389c19 --- /dev/null +++ b/contrib/docker/conf/homeserver.yaml @@ -0,0 +1,181 @@ +# vim:ft=yaml + +## TLS ## + +tls_certificate_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.crt" +tls_private_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.key" +tls_dh_params_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.dh" +no_tls: {{ "True" if SYNAPSE_NO_TLS else "False" }} +tls_fingerprints: [] + +## Server ## + +server_name: "{{ SYNAPSE_SERVER_NAME }}" +pid_file: /homeserver.pid +web_client: {{ "True" if SYNAPSE_WEB_CLIENT else "False" }} +soft_file_limit: 0 + +## Ports ## + +listeners: + {% if not SYNAPSE_NO_TLS %} + - + port: 8448 + bind_addresses: ['0.0.0.0'] + type: http + tls: true + x_forwarded: false + resources: + - names: [client, webclient] + compress: true + - names: [federation] # Federation APIs + compress: false + {% endif %} + + - port: 8008 + tls: false + bind_addresses: ['0.0.0.0'] + type: http + x_forwarded: false + + resources: + - names: [client, webclient] + compress: true + - names: [federation] + compress: false + +## Database ## + +{% if SYNAPSE_DB_HOST %} + +{% else %} +database: + name: "sqlite3" + args: + database: "/data/homeserver.db" +{% endif %} + +## Performance ## + +event_cache_size: "{{ SYNAPSE_EVENT_CACHE_SIZE or "10K" }}" +verbose: 0 +log_file: "/data/homeserver.log" +log_config: "/data/{{ SYNAPSE_SERVER_NAME }}.log.config" + +## Ratelimiting ## + +rc_messages_per_second: 0.2 +rc_message_burst_count: 10.0 +federation_rc_window_size: 1000 +federation_rc_sleep_limit: 10 +federation_rc_sleep_delay: 500 +federation_rc_reject_limit: 50 +federation_rc_concurrent: 3 + +## Files ## + +media_store_path: "/data/media" +uploads_path: "/data/uploads" +max_upload_size: "10M" +max_image_pixels: "32M" +dynamic_thumbnails: false + +# List of thumbnail to precalculate when an image is uploaded. +thumbnail_sizes: +- width: 32 + height: 32 + method: crop +- width: 96 + height: 96 + method: crop +- width: 320 + height: 240 + method: scale +- width: 640 + height: 480 + method: scale +- width: 800 + height: 600 + method: scale + +url_preview_enabled: False +max_spider_size: "10M" + +## Captcha ## + +recaptcha_public_key: "YOUR_PUBLIC_KEY" +recaptcha_private_key: "YOUR_PRIVATE_KEY" +enable_registration_captcha: False +recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify" + +## Turn ## + +turn_uris: [] +turn_shared_secret: "YOUR_SHARED_SECRET" +turn_user_lifetime: "1h" +turn_allow_guests: True + +## Registration ## + +enable_registration: {{ "True" if SYNAPSE_ENABLE_REGISTRATION else "False" }} +registration_shared_secret: "{{ SYNAPSE_REGISTRATION_SHARED_SECRET }}" +bcrypt_rounds: 12 +allow_guest_access: {{ "True" if SYNAPSE_ALLOW_GUEST else "False" }} + +# The list of identity servers trusted to verify third party +# identifiers by this server. +trusted_third_party_id_servers: + - matrix.org + - vector.im + - riot.im + +## Metrics ### + +enable_metrics: False +report_stats: False + +## API Configuration ## + +room_invite_state_types: + - "m.room.join_rules" + - "m.room.canonical_alias" + - "m.room.avatar" + - "m.room.name" + +app_service_config_files: [] +macaroon_secret_key: "{{ SYNAPSE_MACAROON_SECRET_KEY }}" +expire_access_token: False + +## Signing Keys ## + +signing_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.signing.key" +old_signing_keys: {} +key_refresh_interval: "1d" # 1 Day. + +# The trusted servers to download signing keys from. +perspectives: + servers: + "matrix.org": + verify_keys: + "ed25519:auto": + key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw" + +password_config: + enabled: true + +#email: +# enable_notifs: false +# smtp_host: "localhost" +# smtp_port: 25 +# smtp_user: "exampleusername" +# smtp_pass: "examplepassword" +# require_transport_security: False +# notif_from: "Your Friendly %(app)s Home Server " +# app_name: Matrix +# template_dir: res/templates +# notif_template_html: notif_mail.html +# notif_template_text: notif_mail.txt +# notif_for_new_users: True +# riot_base_url: "http://localhost/riot" + +enable_group_creation: true diff --git a/contrib/docker/conf/log.config b/contrib/docker/conf/log.config new file mode 100644 index 0000000000..45e7eef953 --- /dev/null +++ b/contrib/docker/conf/log.config @@ -0,0 +1,36 @@ +version: 1 + +formatters: + precise: + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s- %(message)s' + +filters: + context: + (): synapse.util.logcontext.LoggingContextFilter + request: "" + +handlers: + file: + class: logging.handlers.RotatingFileHandler + formatter: precise + filename: /data/homeserver.log + maxBytes: 104857600 + backupCount: 10 + filters: [context] + console: + class: logging.StreamHandler + formatter: precise + filters: [context] + +loggers: + synapse: + level: INFO + + synapse.storage.SQL: + # beware: increasing this to DEBUG will make synapse log sensitive + # information such as access tokens. + level: INFO + +root: + level: INFO + handlers: [file, console] From 48bc22f89dadb8278cf2b8c940604534999d246f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 10:58:07 +0100 Subject: [PATCH 005/310] Allow for a wheel cache and include missing files in the build --- .dockerignore | 5 +++++ Dockerfile | 13 +++++++------ contrib/docker/start.py | 1 + 3 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..f36f86fbb7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +Dockerfile +.travis.yml +.gitignore +demo/etc +tox.ini diff --git a/Dockerfile b/Dockerfile index 5f0433004f..277246b697 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,15 +2,16 @@ FROM python:2-alpine RUN apk add --no-cache --virtual .nacl_deps build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers -COPY synapse /usr/local/src/synapse -COPY setup.py setup.cfg README.rst synctl /usr/local/src/ +COPY . /synapse -RUN cd /usr/local/src \ - && pip install --upgrade --process-dependency-links . \ +# A wheel cache may be provided in ./cache for faster build +RUN cd /synapse \ + && pip install --upgrade pip setuptools \ + && mkdir -p /synapse/cache \ + && pip install -f /synapse/cache --upgrade --process-dependency-links . \ + && mv /synapse/contrib/docker/* / \ && rm -rf setup.py setup.cfg synapse -COPY contrib/docker / - VOLUME ["/data"] ENTRYPOINT ["/start.py"] diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 4f63ea1ad5..2c427ba1b7 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -22,6 +22,7 @@ if mode == "generate": # Parse the configuration file if not os.path.exists("/compiled"): os.mkdir("/compiled") + convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml") convert("/conf/log.config", "/compiled/%s.log.config" % os.environ.get("SYNAPSE_SERVER_NAME")) From 6d1e28a8426da9e954a3edec25a8717376c583f3 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 11:14:06 +0100 Subject: [PATCH 006/310] Generate any missing keys before starting synapse --- contrib/docker/start.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 2c427ba1b7..e50d23be5f 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -3,7 +3,7 @@ import jinja2 import os import sys -import socket +import subprocess convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) mode = sys.argv[1] if len(sys.argv) > 1 else None @@ -12,19 +12,20 @@ if "SYNAPSE_SERVER_NAME" not in os.environ: print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.") sys.exit(2) -params = ["--server-name", os.environ.get("SYNAPSE_SERVER_NAME"), - "--report-stats", os.environ.get("SYNAPSE_REPORT_STATS", "no"), - "--config-path", os.environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")] - -if mode == "generate": - params.append("--generate-config") +args = ["python", "-m", "synapse.app.homeserver", + "--server-name", os.environ.get("SYNAPSE_SERVER_NAME"), + "--report-stats", os.environ.get("SYNAPSE_REPORT_STATS", "no"), + "--config-path", os.environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")] # Parse the configuration file -if not os.path.exists("/compiled"): - os.mkdir("/compiled") - +if not os.path.exists("/compiled"): os.mkdir("/compiled") convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml") convert("/conf/log.config", "/compiled/%s.log.config" % os.environ.get("SYNAPSE_SERVER_NAME")) -# TODO, replace with a call to synapse.app.homeserver.run() -os.execv("/usr/local/bin/python", ["python", "-m", "synapse.app.homeserver"] + params) +# In generate mode, generate a configuration, missing keys, then exit +if mode == "generate": + os.execv("/usr/local/bin/python", args + ["--generate-config"]) +# In normal mode, generate missing keys if any, then run synapse +else: + subprocess.check_output(args + ["--generate-keys"]) + os.execv("/usr/local/bin/python", args) From f2bf0cda02fef358172033b28dab5f1805c31cad Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 11:40:20 +0100 Subject: [PATCH 007/310] Generate shared secrets if not defined in the environment --- contrib/docker/start.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/contrib/docker/start.py b/contrib/docker/start.py index e50d23be5f..7057f85f61 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -5,10 +5,11 @@ import os import sys import subprocess -convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) +convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(src).read()).render(**environ)) mode = sys.argv[1] if len(sys.argv) > 1 else None +environ = os.environ.copy() -if "SYNAPSE_SERVER_NAME" not in os.environ: +if "SYNAPSE_SERVER_NAME" not in environ: print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.") sys.exit(2) @@ -17,10 +18,16 @@ args = ["python", "-m", "synapse.app.homeserver", "--report-stats", os.environ.get("SYNAPSE_REPORT_STATS", "no"), "--config-path", os.environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")] +# Generate any missing shared secret +for secret in ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KEY"): + if secret not in environ: + print("Generating a random secret for {}".format(secret)) + environ[secret] = os.urandom(32).encode("hex") + # Parse the configuration file if not os.path.exists("/compiled"): os.mkdir("/compiled") -convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml") -convert("/conf/log.config", "/compiled/%s.log.config" % os.environ.get("SYNAPSE_SERVER_NAME")) +convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) +convert("/conf/log.config", "/compiled/%s.log.config" % environ.get("SYNAPSE_SERVER_NAME"), environ) # In generate mode, generate a configuration, missing keys, then exit if mode == "generate": From 886c2d50197bb3558168ce45b6975777780c7aad Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 12:20:29 +0100 Subject: [PATCH 008/310] Support an external postgresql config in the Docker image --- contrib/docker/conf/homeserver.yaml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 851c389c19..1a685320c0 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -46,8 +46,17 @@ listeners: ## Database ## -{% if SYNAPSE_DB_HOST %} - +{% if SYNAPSE_DB_PASSWORD %} +database: + name: "psycopg2" + args: + user: "{{ SYNAPSE_DB_USER or "matrix" }}" + password: "{{ SYNAPSE_DB_PASSWORD }}" + database: "{{ SYNAPSE_DB_DATABASE or "matrix" }}" + host: "{{ SYNAPSE_DB_HOST or "db" }}" + port: "{{ SYNAPSE_DB_PORT or "5432" }}" + cp_min: 5 + cp_max: 10 {% else %} database: name: "sqlite3" From 042757feb25ba9f0cccfdaa6c8775957880c2d2e Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 12:28:42 +0100 Subject: [PATCH 009/310] Install the postgres dependencies --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 277246b697..881c25c243 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,12 @@ FROM python:2-alpine -RUN apk add --no-cache --virtual .nacl_deps build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers +RUN apk add --no-cache --virtual .nacl_deps build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers postgresql-dev COPY . /synapse # A wheel cache may be provided in ./cache for faster build RUN cd /synapse \ - && pip install --upgrade pip setuptools \ + && pip install --upgrade pip setuptools psycopg2 \ && mkdir -p /synapse/cache \ && pip install -f /synapse/cache --upgrade --process-dependency-links . \ && mv /synapse/contrib/docker/* / \ From 1ba2fe114c79dd0f2f2fb8868f8cb0a9f5893652 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 12:55:20 +0100 Subject: [PATCH 010/310] Provide an example docker compose file --- Dockerfile | 2 ++ contrib/docker/docker-compose.yml | 32 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 contrib/docker/docker-compose.yml diff --git a/Dockerfile b/Dockerfile index 881c25c243..25f3746303 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,4 +14,6 @@ RUN cd /synapse \ VOLUME ["/data"] +EXPOSE 8448 + ENTRYPOINT ["/start.py"] diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml new file mode 100644 index 0000000000..659bee2979 --- /dev/null +++ b/contrib/docker/docker-compose.yml @@ -0,0 +1,32 @@ +# This compose file is compatible with Copose itself, it might need some +# adjustments to run properly with stack. + +version: '3' + +services: + + synapse: + image: matrix/synapse + # See the readme for a full documentation of the environment settings + environment: + - SYNAPSE_SERVER_NAME=my.matrix.host + - SYNAPSE_DB_PASSWORD=changeme + volumes: + - ./files:/data + # One may either expose ports directly + ports: + - 8448:8448/tcp + # ... or use a reverse proxy, here is an example for traefik + labels: + - traefik.enable=true + - traefik.frontend.rule=Host:my.matrix.Host + - traefik.port=8448 + + db: + image: postgres:latest + # Change that password, of course! + environment: + - POSTGRES_USER=matrix + - POSTGRES_PASSWORD=changeme + volumes: + - ./schemas:/var/lib/postgres From a207cccb059451682564cddf0e39c7c45b06cb72 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 15:04:26 +0100 Subject: [PATCH 011/310] Reuse environment variables of the postgres container --- contrib/docker/conf/homeserver.yaml | 12 ++++++------ contrib/docker/docker-compose.yml | 11 ++++++++--- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 1a685320c0..7450cc1228 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -46,15 +46,15 @@ listeners: ## Database ## -{% if SYNAPSE_DB_PASSWORD %} +{% if POSTGRES_PASSWORD %} database: name: "psycopg2" args: - user: "{{ SYNAPSE_DB_USER or "matrix" }}" - password: "{{ SYNAPSE_DB_PASSWORD }}" - database: "{{ SYNAPSE_DB_DATABASE or "matrix" }}" - host: "{{ SYNAPSE_DB_HOST or "db" }}" - port: "{{ SYNAPSE_DB_PORT or "5432" }}" + user: "{{ POSTGRES_USER or "matrix" }}" + password: "{{ POSTGRES_PASSWORD }}" + database: "{{ POSTGRES_DB or "matrix" }}" + host: "{{ POSTGRES_HOST or "db" }}" + port: "{{ POSTGRES_PORT or "5432" }}" cp_min: 5 cp_max: 10 {% else %} diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 659bee2979..b8f9741f05 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -6,13 +6,18 @@ version: '3' services: synapse: - image: matrix/synapse + image: synapse + # Since snyapse does not retry to connect to the database, restart upon + # failure + restart: unless-stopped # See the readme for a full documentation of the environment settings environment: - SYNAPSE_SERVER_NAME=my.matrix.host - - SYNAPSE_DB_PASSWORD=changeme + - SYNAPSE_ENABLE_REGISTRATION=yes volumes: - ./files:/data + depends_on: + - db # One may either expose ports directly ports: - 8448:8448/tcp @@ -29,4 +34,4 @@ services: - POSTGRES_USER=matrix - POSTGRES_PASSWORD=changeme volumes: - - ./schemas:/var/lib/postgres + - ./schemas:/var/lib/postgresql/data From 84a9209ba7294243e5bbefede46a3dee7994da9b Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 15:08:43 +0100 Subject: [PATCH 012/310] Remove etc/service files from rob's branch --- contrib/docker/rob/docker-compose.yaml | 39 ---------- .../rob/rootfs/etc/service/synapse/finish | 17 ----- .../docker/rob/rootfs/etc/service/synapse/run | 75 ------------------- 3 files changed, 131 deletions(-) delete mode 100644 contrib/docker/rob/docker-compose.yaml delete mode 100755 contrib/docker/rob/rootfs/etc/service/synapse/finish delete mode 100755 contrib/docker/rob/rootfs/etc/service/synapse/run diff --git a/contrib/docker/rob/docker-compose.yaml b/contrib/docker/rob/docker-compose.yaml deleted file mode 100644 index 73cc29f8fd..0000000000 --- a/contrib/docker/rob/docker-compose.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2017 Vector Creations Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: '3' - -services: - postgres: - image: postgres:9.6.5-alpine - environment: - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_DB: synapse - expose: - - 5432 - restart: unless-stopped - volumes: - - postgres-data:/var/lib/postgresql/data/ - - synapse: - image: matrixdotorg/synapse:v0.22.1 - ports: - - 8008:8008 - - 8448:8448 - restart: unless-stopped - volumes: - - ${CONFIG_PATH}:/synapse/config/ - -volumes: - postgres-data: diff --git a/contrib/docker/rob/rootfs/etc/service/synapse/finish b/contrib/docker/rob/rootfs/etc/service/synapse/finish deleted file mode 100755 index 2aace581a1..0000000000 --- a/contrib/docker/rob/rootfs/etc/service/synapse/finish +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# -# Copyright 2017 Vector Creations Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kill -TERM 1 diff --git a/contrib/docker/rob/rootfs/etc/service/synapse/run b/contrib/docker/rob/rootfs/etc/service/synapse/run deleted file mode 100755 index dd797d3ef9..0000000000 --- a/contrib/docker/rob/rootfs/etc/service/synapse/run +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash -# -# Copyright 2017 Vector Creations Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -: ${CONFIG_PATH:="/synapse/config"} -: ${POSTGRES_DATABASE:="synapse"} -: ${POSTGRES_HOST:="postgres"} -: ${POSTGRES_USER:="postgres"} -: ${REPORT_STATS:="yes"} -: ${SERVER_NAME:="localhost"} - -DATABASE_CONFIG_PATH="${CONFIG_PATH}/database.yaml" -HOMESERVER_CONFIG_PATH="${CONFIG_PATH}/homeserver.yaml" -SYNAPSE_COMMAND="python -m synapse.app.homeserver" - -. /synapse/bin/activate -cd /synapse - -if [[ -n "${GENERATE_CONFIG}" ]]; then - ${SYNAPSE_COMMAND} \ - --server-name ${SERVER_NAME} \ - --config-path ${HOMESERVER_CONFIG_PATH} \ - --generate-config \ - --report-stats=${REPORT_STATS} - - if [[ -f "${DATABASE_CONFIG_PATH}" ]]; then - echo "Config file '${DATABASE_CONFIG_PATH}' already exists. Remove it if you want it to be generated." - else - echo "Generating ${DATABASE_CONFIG_PATH}..." - if [[ -n "${POSTGRES_PASSWORD}" ]]; then - (cat > ${DATABASE_CONFIG_PATH}) < ${DATABASE_CONFIG_PATH}) < Date: Sun, 4 Feb 2018 15:27:32 +0100 Subject: [PATCH 013/310] Update sumperdump Docker readme to match this image properties --- contrib/docker/README.md | 108 ++++++++++++++++++++++++--------------- 1 file changed, 67 insertions(+), 41 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index c15517d0e0..8ba5f79692 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -1,5 +1,12 @@ # Synapse Docker +This Docker image will run Synapse as a single process. It does not provide any +database server or TURN server that you should run separately. + +If you run a Postgres server, you should simply have it in the same Compose +project or set the proper environment variables and the image will automatically +use that server. + ## Build Build the docker image with the `docker build` command from the root of the synapse repository. @@ -10,54 +17,29 @@ docker build -t matrixdotorg/synapse:v0.22.1 . The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:` where `` is the same as the release tag in the synapse git repository. -## Configure - -Synapse provides a command for generating homeserver configuration files. These are a good starting point for setting up your own deployment. - -The documentation below will refer to a `CONFIG_PATH` shell variable. This is a path to a directory where synapse configuration will be stored. It needs to be mapped into the container as a volume at `/synapse/config/` as can be seen in the example `docker run` command. - -Docker container environment variables: -* `GENERATE_CONFIG` - Set this to any non-empty string, such as `yes`, to trigger generation of configuration files. Existing files in the `CONFIG_PATH` will **not** be overwritten. -* `POSTGRES_DATABASE` - The database name for the synapse postgres database. [default: `synapse`] -* `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `postgres` which is useful when using a container on the same docker network in a compose file where the postgres service is called `postgres`] **NOTE**: `localhost` and `127.0.0.1` refer to the container itself unless running the container with `host` networking. -* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy. -* `POSTGRES_USER` - The user for the synapse postgres database. [default: `postgres`] -* `REPORT_STATS` - Whether to send anonymous usage statistics back to the Matrix project which helps us to get funding! Must be `yes` or `no`. [default: `yes`] -* `SERVER_NAME` - The domain used for the Matrix homeserver. If you intend to run this synapse instance on a public domain, use that domain. [default: `localhost`] - -``` -CONFIG_PATH=/my/magical/config/path/ -mkdir -p ${CONFIG_PATH} -docker run \ - --rm \ - -e GENERATE_CONFIG=yes \ - -e POSTGRES_PASSWORD=MyVerySecretPassword \ - -e REPORT_STATS=yes \ - -e SERVER_NAME=example.com \ - -v ${CONFIG_PATH}:/synapse/config/ \ - matrixdotorg/synapse:v0.22.1 -``` - -This will create a temporary container from the image and use the synapse code for generating configuration files and TLS keys and certificates for the specified `SERVER_NAME` domain. The files are written to `CONFIG_PATH`. +You may have a local Python wheel cache available, in which case copy the relevant packages in the ``cache/`` directory at the root of the project. ## Run -**NOTE**: If you are not using postgresql and are using sqlite3 as your database, you will need to make a directory to store the sqlite3 database file in and then mount this volume into the container at `/synapse/data/`. As it is so easy to use postgresql, when using Docker containers, this is not documented to somewhat discourage it. Choose a `POSTGRES_PASSWORD` instead. +It is recommended that you use Docker Compose to run your containers, including +this image and a Postgres server. A sample ``docker-compose.yml`` is provided, +with example labels for a reverse proxy and other artifacts. -### Docker Compose +Then, to run the server: -A `docker-compose.yaml` file is included to ease deployment of the basic synapse and postgres setup. Remember to set a `POSTGRES_PASSWORD` when generating your configuration above. You will need it for running the containers in the composition. - -From the `docker/` subdirectory of the synapse repository: ``` -CONFIG_PATH=/my/magical/config/path/ -POSTGRES_PASSWORD=MyVerySecretPassword \ -docker-compose \ - -p synapse \ - up -d +docker-compose up -d ``` -### Docker +In the case you specified a custom path for you configuration file and wish to +generate a fresh ``homeserver.yaml``, simply run: + +``` +docker-compose run synapse generate +``` + +If you do not wish to use Compose, you may still run this image using plain +Docker commands: Note that the following is just a guideline and you may need to add parameters to the docker run command to account for the network situation with your postgres database. @@ -65,6 +47,50 @@ Note that the following is just a guideline and you may need to add parameters t docker run \ -d \ --name synapse \ - -v ${CONFIG_PATH}:/synapse/config/ \ + -v ${DATA_PATH}:/data \ + -e SYNAPSE_SERVER_NAME=my.matrix.host \ matrixdotorg/synapse:v0.22.1 ``` + + +## Volumes + +The image expects a single volue, located at ``/data``, that will hold: + +* temporary files during uploads; +* uploaded media and thumbnais; +* the SQLite database if you do not configure postgres. + +## Environment + +If you do not specify a custom path for the configuration file, a very generic +file will be generated, based on the following environment settings. +These are a good starting point for setting up your own deployment. + +Synapse specific settings: + +* ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname. +* ``SYNAPSE_CONFIG_PATH``, path to a custom config file (will ignore all + other options then). +* ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if + you run your own TLS-capable reverse proxy). +* ``SYNAPSE_WEB_CLIENT``, set this variable to enable the embedded Web client. +* ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on + the Synapse instance. +* ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server. +* ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`]. +* ``SYNAPSE_REPORT_STATS``, set this variable to `yes` to enable anonymous + statistics reporting back to the Matrix project which helps us to get funding. + +Shared secrets, these will be initialized to random values if not set: + +* ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if + registration is disable. +* ``SYNAPSE_MACAROON_SECRET_KEY``, secret for Macaroon. + +Database specific values (will use SQLite if not set): + +* `POSTGRES_DATABASE` - The database name for the synapse postgres database. [default: `matrix`] +* `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `db` which is useful when using a container on the same docker network in a compose file where the postgres service is called `db`] +* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy. +* `POSTGRES_USER` - The user for the synapse postgres database. [default: `matrix`] From b8ab78b82c60ab9c897891cf9f53c9cc44873a25 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 15:41:54 +0100 Subject: [PATCH 014/310] Add the build cache/ folder to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 491047c352..d13017f400 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ demo/media_store.* demo/etc uploads +cache .idea/ media_store/ From f72c9c1fb650c7f3bac4d77d4e24fa0469d09ebb Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 16:18:40 +0100 Subject: [PATCH 015/310] Fix multiple typos --- MANIFEST.in | 1 + contrib/docker/README.md | 19 ++++++++++++------- contrib/docker/docker-compose.yml | 7 ++++--- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index afb60e12ee..8c9a57a9ca 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -25,6 +25,7 @@ recursive-include synapse/static *.js exclude jenkins.sh exclude jenkins*.sh exclude jenkins* +exclude Dockerfile recursive-exclude jenkins *.sh prune .github diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 8ba5f79692..73e53e4306 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -23,7 +23,7 @@ You may have a local Python wheel cache available, in which case copy the releva It is recommended that you use Docker Compose to run your containers, including this image and a Postgres server. A sample ``docker-compose.yml`` is provided, -with example labels for a reverse proxy and other artifacts. +including example labels for reverse proxying and other artifacts. Then, to run the server: @@ -35,13 +35,13 @@ In the case you specified a custom path for you configuration file and wish to generate a fresh ``homeserver.yaml``, simply run: ``` -docker-compose run synapse generate +docker-compose run --rm synapse generate ``` If you do not wish to use Compose, you may still run this image using plain -Docker commands: - -Note that the following is just a guideline and you may need to add parameters to the docker run command to account for the network situation with your postgres database. +Docker commands. Note that the following is just a guideline and you may need +to add parameters to the docker run command to account for the network situation +with your postgres database. ``` docker run \ @@ -55,7 +55,7 @@ docker run \ ## Volumes -The image expects a single volue, located at ``/data``, that will hold: +The image expects a single volume, located at ``/data``, that will hold: * temporary files during uploads; * uploaded media and thumbnais; @@ -63,10 +63,15 @@ The image expects a single volue, located at ``/data``, that will hold: ## Environment -If you do not specify a custom path for the configuration file, a very generic +Unless you specify a custom path for the configuration file, a very generic file will be generated, based on the following environment settings. These are a good starting point for setting up your own deployment. +Global settings: + +* ``UID``, the user id Synapse will run as [default 991] +* ``GID``, the group id Synapse will run as [default 991] + Synapse specific settings: * ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname. diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index b8f9741f05..727743effa 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -1,4 +1,4 @@ -# This compose file is compatible with Copose itself, it might need some +# This compose file is compatible with Compose itself, it might need some # adjustments to run properly with stack. version: '3' @@ -18,10 +18,11 @@ services: - ./files:/data depends_on: - db - # One may either expose ports directly + # In order to expose Synapse, remove one of the following, you might for + # instance expose the TLS port directly: ports: - 8448:8448/tcp - # ... or use a reverse proxy, here is an example for traefik + # ... or use a reverse proxy, here is an example for traefik: labels: - traefik.enable=true - traefik.frontend.rule=Host:my.matrix.Host From e9021e16c49c4224782040449b43fb0015c9f05c Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 23:19:08 +0100 Subject: [PATCH 016/310] Run the server as an unprivileged user --- Dockerfile | 2 +- contrib/docker/start.py | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 25f3746303..f687a4f2bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM python:2-alpine -RUN apk add --no-cache --virtual .nacl_deps build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers postgresql-dev +RUN apk add --no-cache --virtual .nacl_deps su-exec build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers postgresql-dev COPY . /synapse diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 7057f85f61..8bc72bf428 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -9,14 +9,16 @@ convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(sr mode = sys.argv[1] if len(sys.argv) > 1 else None environ = os.environ.copy() +# Check mandatory parameters and build the base start arguments if "SYNAPSE_SERVER_NAME" not in environ: print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.") sys.exit(2) +permissions = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991)) args = ["python", "-m", "synapse.app.homeserver", - "--server-name", os.environ.get("SYNAPSE_SERVER_NAME"), - "--report-stats", os.environ.get("SYNAPSE_REPORT_STATS", "no"), - "--config-path", os.environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")] + "--server-name", environ.get("SYNAPSE_SERVER_NAME"), + "--report-stats", environ.get("SYNAPSE_REPORT_STATS", "no"), + "--config-path", environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")] # Generate any missing shared secret for secret in ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KEY"): @@ -35,4 +37,5 @@ if mode == "generate": # In normal mode, generate missing keys if any, then run synapse else: subprocess.check_output(args + ["--generate-keys"]) - os.execv("/usr/local/bin/python", args) + subprocess.check_output(["chown", "-R", permissions, "/data"]) + os.execv("/sbin/su-exec", ["su-exec", permissions] + args) From 8db84e9b2198ad4a80f6e9e66e2fc8c2d44fbeca Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 20:08:35 +0100 Subject: [PATCH 017/310] Remove docker related files from the python manifest --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 8c9a57a9ca..e2a6623a63 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -26,6 +26,7 @@ exclude jenkins.sh exclude jenkins*.sh exclude jenkins* exclude Dockerfile +exclude .dockerignore recursive-exclude jenkins *.sh prune .github From 81010a126e9bbcce018b104921ff5221a67f99cc Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 21:28:15 +0100 Subject: [PATCH 018/310] Add dynamic recaptcha configuration in the Docker image --- contrib/docker/README.md | 4 ++++ contrib/docker/conf/homeserver.yaml | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 73e53e4306..f4bc78908d 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -86,6 +86,10 @@ Synapse specific settings: * ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`]. * ``SYNAPSE_REPORT_STATS``, set this variable to `yes` to enable anonymous statistics reporting back to the Matrix project which helps us to get funding. +* ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public + key in order to enable recaptcha upon registration +* ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private + key in order to enable recaptcha upon registration Shared secrets, these will be initialized to random values if not set: diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 7450cc1228..6f1de24aad 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -112,10 +112,17 @@ max_spider_size: "10M" ## Captcha ## +{% if SYNAPSE_RECAPTCHA_PUBLIC_KEY %} +recaptcha_public_key: "{{ SYNAPSE_RECAPTCHA_PUBLIC_KEY }}" +recaptcha_private_key: "{{ SYNAPSE_RECAPTCHA_PRIVATE_KEY }}" +enable_registration_captcha: True +recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify" +{% else %} recaptcha_public_key: "YOUR_PUBLIC_KEY" recaptcha_private_key: "YOUR_PRIVATE_KEY" enable_registration_captcha: False recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify" +{% endif %} ## Turn ## From cd51931b62aef63dacf4d79cdfa5de56da4eeda6 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 21:53:53 +0100 Subject: [PATCH 019/310] Add dynamic TURN configuration in the Docker image --- contrib/docker/README.md | 7 +++++-- contrib/docker/conf/homeserver.yaml | 9 +++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index f4bc78908d..0da7b56628 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -87,9 +87,12 @@ Synapse specific settings: * ``SYNAPSE_REPORT_STATS``, set this variable to `yes` to enable anonymous statistics reporting back to the Matrix project which helps us to get funding. * ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public - key in order to enable recaptcha upon registration + key in order to enable recaptcha upon registration. * ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private - key in order to enable recaptcha upon registration + key in order to enable recaptcha upon registration. +* ``SYNAPSE_TURN_URIS``, set this variable to the coma-separated list of TURN + uris to enable TURN for this homeserver. +* ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required. Shared secrets, these will be initialized to random values if not set: diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 6f1de24aad..6f8fb24e5f 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -126,10 +126,19 @@ recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify" ## Turn ## +{% if SYNAPSE_TURN_URIS %} +turn_uris: +{% for uri in SYNAPSE_TURN_URIS.split(',') %} - {{ uri }} +{% endfor %} +turn_shared_secret: "{{ SYNAPSE_TURN_SECRET }}" +turn_user_lifetime: "1h" +turn_allow_guests: True +{% else %} turn_uris: [] turn_shared_secret: "YOUR_SHARED_SECRET" turn_user_lifetime: "1h" turn_allow_guests: True +{% endif %} ## Registration ## From cf4ef60e287a6a61844ab260606721db2b7ee0cd Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 22:10:03 +0100 Subject: [PATCH 020/310] Document the cache factor environment variable for Docker --- contrib/docker/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 0da7b56628..8b1e0afee6 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -84,6 +84,7 @@ Synapse specific settings: the Synapse instance. * ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server. * ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`]. +* ``SYNAPSE_CACHE_FACTOR``, the cache factor [default `0.5`]. * ``SYNAPSE_REPORT_STATS``, set this variable to `yes` to enable anonymous statistics reporting back to the Matrix project which helps us to get funding. * ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public From d8c7da5dca907bf65293b6b967200141cad69410 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 22:12:50 +0100 Subject: [PATCH 021/310] Fix a typo in the Docker README --- contrib/docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 8b1e0afee6..624dca747a 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -58,7 +58,7 @@ docker run \ The image expects a single volume, located at ``/data``, that will hold: * temporary files during uploads; -* uploaded media and thumbnais; +* uploaded media and thumbnails; * the SQLite database if you do not configure postgres. ## Environment From f5364b47ec4b67a552976b9dbb9594d448744e30 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 22:14:40 +0100 Subject: [PATCH 022/310] Point to the 'latest' tag in the Docker documentation --- contrib/docker/README.md | 2 +- contrib/docker/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 624dca747a..87354b9bc3 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -49,7 +49,7 @@ docker run \ --name synapse \ -v ${DATA_PATH}:/data \ -e SYNAPSE_SERVER_NAME=my.matrix.host \ - matrixdotorg/synapse:v0.22.1 + matrixdotorg/synapse:latest ``` diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 727743effa..3fb156db47 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -6,7 +6,7 @@ version: '3' services: synapse: - image: synapse + image: matrixdotorg/synapse:latest # Since snyapse does not retry to connect to the database, restart upon # failure restart: unless-stopped From 630573a9325f826e81f04650fd83e64d1c831035 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 22:57:22 +0100 Subject: [PATCH 023/310] Do not copy documentation files to the Docker root folder --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f687a4f2bb..2ae503ce79 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN cd /synapse \ && pip install --upgrade pip setuptools psycopg2 \ && mkdir -p /synapse/cache \ && pip install -f /synapse/cache --upgrade --process-dependency-links . \ - && mv /synapse/contrib/docker/* / \ + && mv /synapse/contrib/docker/start.py /synapse/contrib/docker/conf / \ && rm -rf setup.py setup.cfg synapse VOLUME ["/data"] From ee3b160a2ad375223b4304184304605a35f1b406 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 22:57:35 +0100 Subject: [PATCH 024/310] Only generate configuration files when necessary --- contrib/docker/start.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 8bc72bf428..d3364e4226 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -26,16 +26,18 @@ for secret in ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KE print("Generating a random secret for {}".format(secret)) environ[secret] = os.urandom(32).encode("hex") -# Parse the configuration file -if not os.path.exists("/compiled"): os.mkdir("/compiled") -convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) -convert("/conf/log.config", "/compiled/%s.log.config" % environ.get("SYNAPSE_SERVER_NAME"), environ) - # In generate mode, generate a configuration, missing keys, then exit if mode == "generate": os.execv("/usr/local/bin/python", args + ["--generate-config"]) + # In normal mode, generate missing keys if any, then run synapse else: + # Parse the configuration file + if "SYNAPSE_CONFIG_PATH" not in environ: + if not os.path.exists("/compiled"): os.mkdir("/compiled") + convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) + convert("/conf/log.config", "/compiled/%s.log.config" % environ.get("SYNAPSE_SERVER_NAME"), environ) + # Generate missing keys and start synapse subprocess.check_output(args + ["--generate-keys"]) subprocess.check_output(["chown", "-R", permissions, "/data"]) os.execv("/sbin/su-exec", ["su-exec", permissions] + args) From 107a5c94418de55d52a0232115d5c3c7efc6d285 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 23:02:33 +0100 Subject: [PATCH 025/310] Add the non-tls port to the expose list --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2ae503ce79..bec884ce9b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,6 @@ RUN cd /synapse \ VOLUME ["/data"] -EXPOSE 8448 +EXPOSE 8008/tcp 8448/tcp ENTRYPOINT ["/start.py"] From 1ffd9cb93617fe9bb2367d575786c0ff222cd415 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 23:13:27 +0100 Subject: [PATCH 026/310] Support loading application service files from /data/appservices/ --- contrib/docker/README.md | 7 ++++++- contrib/docker/conf/homeserver.yaml | 9 ++++++++- contrib/docker/start.py | 4 ++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 87354b9bc3..c1724fe269 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -59,7 +59,12 @@ The image expects a single volume, located at ``/data``, that will hold: * temporary files during uploads; * uploaded media and thumbnails; -* the SQLite database if you do not configure postgres. +* the SQLite database if you do not configure postgres; +* the appservices configuration. + +In order to setup an application service, simply create an ``appservices`` +directory in the data volume and write the application service Yaml +configuration file there. Multiple application services are supported. ## Environment diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 6f8fb24e5f..e5d3f965e4 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -128,7 +128,7 @@ recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify" {% if SYNAPSE_TURN_URIS %} turn_uris: -{% for uri in SYNAPSE_TURN_URIS.split(',') %} - {{ uri }} +{% for uri in SYNAPSE_TURN_URIS.split(',') %} - "{{ uri }}" {% endfor %} turn_shared_secret: "{{ SYNAPSE_TURN_SECRET }}" turn_user_lifetime: "1h" @@ -167,7 +167,14 @@ room_invite_state_types: - "m.room.avatar" - "m.room.name" +{% if SYNAPSE_APPSERVICES %} +app_service_config_files: +{% for appservice in SYNAPSE_APPSERVICES %} - "{{ appservice }}" +{% endfor %} +{% else %} app_service_config_files: [] +{% endif %} + macaroon_secret_key: "{{ SYNAPSE_MACAROON_SECRET_KEY }}" expire_access_token: False diff --git a/contrib/docker/start.py b/contrib/docker/start.py index d3364e4226..8ade0f227d 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -4,6 +4,7 @@ import jinja2 import os import sys import subprocess +import glob convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(src).read()).render(**environ)) mode = sys.argv[1] if len(sys.argv) > 1 else None @@ -26,6 +27,9 @@ for secret in ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KE print("Generating a random secret for {}".format(secret)) environ[secret] = os.urandom(32).encode("hex") +# Load appservices configurations +environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") + # In generate mode, generate a configuration, missing keys, then exit if mode == "generate": os.execv("/usr/local/bin/python", args + ["--generate-config"]) From 63fd148724399d52f3435b1c172435d7cabcde4c Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 19:46:11 +0100 Subject: [PATCH 027/310] Make it clear that two modes are avaiable in the documentation, improve the compose file --- contrib/docker/README.md | 17 ++++++++++------- contrib/docker/docker-compose.yml | 12 ++++++++++-- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index c1724fe269..197bad103e 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -12,7 +12,7 @@ use that server. Build the docker image with the `docker build` command from the root of the synapse repository. ``` -docker build -t matrixdotorg/synapse:v0.22.1 . +docker build -t matrixdotorg/synapse . ``` The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:` where `` is the same as the release tag in the synapse git repository. @@ -76,12 +76,17 @@ Global settings: * ``UID``, the user id Synapse will run as [default 991] * ``GID``, the group id Synapse will run as [default 991] +* ``SYNAPSE_CONFIG_PATH``, path to a custom config file -Synapse specific settings: +If ``SYNAPSE_CONFIG_PATH`` is set, you should generate a configuration file +then customize it manually. No other environment variable is required. + +Otherwise, a dynamic configuration file will be used. The following environment +variables are available for configuration: * ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname. -* ``SYNAPSE_CONFIG_PATH``, path to a custom config file (will ignore all - other options then). +* ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``not``), enable anonymous + statistics reporting back to the Matrix project which helps us to get funding. * ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if you run your own TLS-capable reverse proxy). * ``SYNAPSE_WEB_CLIENT``, set this variable to enable the embedded Web client. @@ -90,8 +95,6 @@ Synapse specific settings: * ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server. * ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`]. * ``SYNAPSE_CACHE_FACTOR``, the cache factor [default `0.5`]. -* ``SYNAPSE_REPORT_STATS``, set this variable to `yes` to enable anonymous - statistics reporting back to the Matrix project which helps us to get funding. * ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public key in order to enable recaptcha upon registration. * ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private @@ -100,7 +103,7 @@ Synapse specific settings: uris to enable TURN for this homeserver. * ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required. -Shared secrets, these will be initialized to random values if not set: +Shared secrets, that will be initialized to random values if not set: * ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if registration is disable. diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 3fb156db47..b07984ea34 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -6,7 +6,7 @@ version: '3' services: synapse: - image: matrixdotorg/synapse:latest + image: docker.io/matrixdotorg/synapse:latest # Since snyapse does not retry to connect to the database, restart upon # failure restart: unless-stopped @@ -15,7 +15,12 @@ services: - SYNAPSE_SERVER_NAME=my.matrix.host - SYNAPSE_ENABLE_REGISTRATION=yes volumes: + # You may either store all the files in a local folder - ./files:/data + # .. or you may split this between different storage points + # - ./files:/data + # - /path/to/ssd:/data/uploads + # - /path/to/large_hdd:/data/media depends_on: - db # In order to expose Synapse, remove one of the following, you might for @@ -29,10 +34,13 @@ services: - traefik.port=8448 db: - image: postgres:latest + image: postgres:10-alpine # Change that password, of course! environment: - POSTGRES_USER=matrix - POSTGRES_PASSWORD=changeme volumes: + # You may store the database tables in a local folder.. - ./schemas:/var/lib/postgresql/data + # .. or store them on some high performance storage for better results + # - /path/to/ssd/storage:/var/lib/postfesql/data From 58df3a8c5dcf5de12bcf9190551dc8241d32b8a1 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 19:48:53 +0100 Subject: [PATCH 028/310] Add some documentation about high performance storage --- contrib/docker/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 197bad103e..3710afb0cf 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -31,11 +31,11 @@ Then, to run the server: docker-compose up -d ``` -In the case you specified a custom path for you configuration file and wish to +In the case you specify a custom path for you configuration file and wish to generate a fresh ``homeserver.yaml``, simply run: ``` -docker-compose run --rm synapse generate +docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host synapse generate ``` If you do not wish to use Compose, you may still run this image using plain @@ -62,6 +62,11 @@ The image expects a single volume, located at ``/data``, that will hold: * the SQLite database if you do not configure postgres; * the appservices configuration. +You are free to use separate volumes depending on storage endpoints at your +disposal. For instance, ``/data/media`` coud be stored on a large but low +performance hdd storage while other files could be stored on high performance +endpoints. + In order to setup an application service, simply create an ``appservices`` directory in the data volume and write the application service Yaml configuration file there. Multiple application services are supported. From 084afbb6a06f2661bed503bf49b0291ce999c6c1 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 19:50:04 +0100 Subject: [PATCH 029/310] Rename the permissions variable to avoid confusion --- contrib/docker/start.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 8ade0f227d..13a10a11bb 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -15,7 +15,7 @@ if "SYNAPSE_SERVER_NAME" not in environ: print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.") sys.exit(2) -permissions = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991)) +ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991)) args = ["python", "-m", "synapse.app.homeserver", "--server-name", environ.get("SYNAPSE_SERVER_NAME"), "--report-stats", environ.get("SYNAPSE_REPORT_STATS", "no"), @@ -43,5 +43,5 @@ else: convert("/conf/log.config", "/compiled/%s.log.config" % environ.get("SYNAPSE_SERVER_NAME"), environ) # Generate missing keys and start synapse subprocess.check_output(args + ["--generate-keys"]) - subprocess.check_output(["chown", "-R", permissions, "/data"]) - os.execv("/sbin/su-exec", ["su-exec", permissions] + args) + subprocess.check_output(["chown", "-R", ownership, "/data"]) + os.execv("/sbin/su-exec", ["su-exec", ownership] + args) From b8a4dceb3cee6b69d1b1b882cef1f96a3ff6249f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 20:41:41 +0100 Subject: [PATCH 030/310] Refactor the start script to better handle mandatory parameters --- contrib/docker/README.md | 2 +- contrib/docker/docker-compose.yml | 1 + contrib/docker/start.py | 56 ++++++++++++++++++------------- 3 files changed, 35 insertions(+), 24 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 3710afb0cf..0493d2ee6e 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -90,7 +90,7 @@ Otherwise, a dynamic configuration file will be used. The following environment variables are available for configuration: * ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname. -* ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``not``), enable anonymous +* ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous statistics reporting back to the Matrix project which helps us to get funding. * ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if you run your own TLS-capable reverse proxy). diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index b07984ea34..3d0b3c0ea4 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -13,6 +13,7 @@ services: # See the readme for a full documentation of the environment settings environment: - SYNAPSE_SERVER_NAME=my.matrix.host + - SYNAPSE_REPORT_STATS=no - SYNAPSE_ENABLE_REGISTRATION=yes volumes: # You may either store all the files in a local folder diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 13a10a11bb..32142bbe00 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -6,42 +6,52 @@ import sys import subprocess import glob +# Utility functions convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(src).read()).render(**environ)) + +def check_arguments(environ, args): + for argument in args: + if argument not in environ: + print("Environment variable %s is mandatory, exiting." % argument) + sys.exit(2) + +def generate_secrets(environ, secrets): + for secret in secrets: + if secret not in environ: + print("Generating a random secret for {}".format(secret)) + environ[secret] = os.urandom(32).encode("hex") + +# Prepare the configuration mode = sys.argv[1] if len(sys.argv) > 1 else None environ = os.environ.copy() - -# Check mandatory parameters and build the base start arguments -if "SYNAPSE_SERVER_NAME" not in environ: - print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.") - sys.exit(2) - ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991)) -args = ["python", "-m", "synapse.app.homeserver", - "--server-name", environ.get("SYNAPSE_SERVER_NAME"), - "--report-stats", environ.get("SYNAPSE_REPORT_STATS", "no"), - "--config-path", environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")] - -# Generate any missing shared secret -for secret in ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KEY"): - if secret not in environ: - print("Generating a random secret for {}".format(secret)) - environ[secret] = os.urandom(32).encode("hex") - -# Load appservices configurations -environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") +args = ["python", "-m", "synapse.app.homeserver"] # In generate mode, generate a configuration, missing keys, then exit if mode == "generate": - os.execv("/usr/local/bin/python", args + ["--generate-config"]) + check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS", "SYNAPSE_CONFIG_PATH")) + args += [ + "--server-name", environ["SYNAPSE_SERVER_NAME"], + "--report-stats", environ["SYNAPSE_REPORT_STATS"], + "--config-path", environ["SYNAPSE_CONFIG_PATH"], + "--generate-config" + ] + os.execv("/usr/local/bin/python", args) # In normal mode, generate missing keys if any, then run synapse else: # Parse the configuration file - if "SYNAPSE_CONFIG_PATH" not in environ: + if "SYNAPSE_CONFIG_PATH" in environ: + args += ["--config-path", environ["SYNAPSE_CONFIG_PATH"]] + else: + check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS")) + generate_secrets(environ, ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KEY")) + environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") if not os.path.exists("/compiled"): os.mkdir("/compiled") convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) - convert("/conf/log.config", "/compiled/%s.log.config" % environ.get("SYNAPSE_SERVER_NAME"), environ) + convert("/conf/log.config", "/compiled/%s.log.config" % environ["SYNAPSE_SERVER_NAME"], environ) + subprocess.check_output(["chown", "-R", ownership, "/data"]) + args += ["--config-path", "/compiled/homeserver.yaml"] # Generate missing keys and start synapse subprocess.check_output(args + ["--generate-keys"]) - subprocess.check_output(["chown", "-R", ownership, "/data"]) os.execv("/sbin/su-exec", ["su-exec", ownership] + args) From e174c46a295ca6e06b217b5dcbcf995b890e6d07 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 20:42:57 +0100 Subject: [PATCH 031/310] Use 'synapse' as a default postgres user in Docker examples --- contrib/docker/conf/homeserver.yaml | 4 ++-- contrib/docker/docker-compose.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index e5d3f965e4..1ca1fe991f 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -50,9 +50,9 @@ listeners: database: name: "psycopg2" args: - user: "{{ POSTGRES_USER or "matrix" }}" + user: "{{ POSTGRES_USER or "synapse" }}" password: "{{ POSTGRES_PASSWORD }}" - database: "{{ POSTGRES_DB or "matrix" }}" + database: "{{ POSTGRES_DB or "synapse" }}" host: "{{ POSTGRES_HOST or "db" }}" port: "{{ POSTGRES_PORT or "5432" }}" cp_min: 5 diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 3d0b3c0ea4..e447bf1212 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -38,7 +38,7 @@ services: image: postgres:10-alpine # Change that password, of course! environment: - - POSTGRES_USER=matrix + - POSTGRES_USER=synapse - POSTGRES_PASSWORD=changeme volumes: # You may store the database tables in a local folder.. From 914a59cb8c12b25d77ed9a81e4543c23d7e10b5e Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 20:43:45 +0100 Subject: [PATCH 032/310] Disable the Web client in the Docker image --- contrib/docker/README.md | 1 - contrib/docker/conf/homeserver.yaml | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 0493d2ee6e..9f40dc0d58 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -94,7 +94,6 @@ variables are available for configuration: statistics reporting back to the Matrix project which helps us to get funding. * ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if you run your own TLS-capable reverse proxy). -* ``SYNAPSE_WEB_CLIENT``, set this variable to enable the embedded Web client. * ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on the Synapse instance. * ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server. diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 1ca1fe991f..19a2cbad29 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -12,7 +12,7 @@ tls_fingerprints: [] server_name: "{{ SYNAPSE_SERVER_NAME }}" pid_file: /homeserver.pid -web_client: {{ "True" if SYNAPSE_WEB_CLIENT else "False" }} +web_client: False soft_file_limit: 0 ## Ports ## @@ -26,7 +26,7 @@ listeners: tls: true x_forwarded: false resources: - - names: [client, webclient] + - names: [client] compress: true - names: [federation] # Federation APIs compress: false @@ -39,7 +39,7 @@ listeners: x_forwarded: false resources: - - names: [client, webclient] + - names: [client] compress: true - names: [federation] compress: false From a0af0054ec91e92a6843c121cd27e92ea63c1034 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 20:46:11 +0100 Subject: [PATCH 033/310] Honor the SYNAPSE_REPORT_STATS parameter in the Docker image --- contrib/docker/conf/homeserver.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 19a2cbad29..3b57f7174d 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -156,8 +156,13 @@ trusted_third_party_id_servers: ## Metrics ### +{% if SYNAPSE_REPORT_STATS.lower() == "yes" %} +enable_metrics: True +report_stats: True +{% else %} enable_metrics: False report_stats: False +{% endif %} ## API Configuration ## From ef1f8d4be6e970043b8283f5caa1ca764652ad56 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 20:53:12 +0100 Subject: [PATCH 034/310] Enable email server configuration from environment variables --- contrib/docker/README.md | 7 ++++++ contrib/docker/conf/homeserver.yaml | 33 +++++++++++++++-------------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 9f40dc0d58..b74c72698c 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -119,3 +119,10 @@ Database specific values (will use SQLite if not set): * `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `db` which is useful when using a container on the same docker network in a compose file where the postgres service is called `db`] * `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy. * `POSTGRES_USER` - The user for the synapse postgres database. [default: `matrix`] + +Mail server specific values (will not send emails if not set): + +* ``SYNAPSE_SMTP_HOST``, hostname to the mail server. +* ``SYNAPSE_SMTP_PORT``, TCP port for accessing the mail server [default ``25``]. +* ``SYNAPSE_SMTP_USER``, username for authenticating against the mail server if any. +* ``SYNAPSE_SMTP_PASSWORD``, password for authenticating against the mail server if any. diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 3b57f7174d..198b8ddee7 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -146,6 +146,7 @@ enable_registration: {{ "True" if SYNAPSE_ENABLE_REGISTRATION else "False" }} registration_shared_secret: "{{ SYNAPSE_REGISTRATION_SHARED_SECRET }}" bcrypt_rounds: 12 allow_guest_access: {{ "True" if SYNAPSE_ALLOW_GUEST else "False" }} +enable_group_creation: true # The list of identity servers trusted to verify third party # identifiers by this server. @@ -200,19 +201,19 @@ perspectives: password_config: enabled: true -#email: -# enable_notifs: false -# smtp_host: "localhost" -# smtp_port: 25 -# smtp_user: "exampleusername" -# smtp_pass: "examplepassword" -# require_transport_security: False -# notif_from: "Your Friendly %(app)s Home Server " -# app_name: Matrix -# template_dir: res/templates -# notif_template_html: notif_mail.html -# notif_template_text: notif_mail.txt -# notif_for_new_users: True -# riot_base_url: "http://localhost/riot" - -enable_group_creation: true +{% if SYNAPSE_SMTP_HOST %} +email: + enable_notifs: false + smtp_host: "{{ SYNAPSE_SMTP_HOST }}" + smtp_port: {{ SYNAPSE_SMTP_PORT or "25" }} + smtp_user: "{{ SYNAPSE_SMTP_USER }}" + smtp_pass: "{{ SYNAPSE_SMTP_PASSWORD }}" + require_transport_security: False + notif_from: "{{ SYNAPSE_SMTP_FROM or "hostmaster@" + SYNAPSE_SERVER_NAME }}" + app_name: Matrix + template_dir: res/templates + notif_template_html: notif_mail.html + notif_template_text: notif_mail.txt + notif_for_new_users: True + riot_base_url: "https://{{ SYNAPSE_SERVER_NAME }}" +{% endif %} From b9b668e4bb6ec3c0ae6c696a4b7abf6f537e0504 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 21:39:36 +0100 Subject: [PATCH 035/310] Update to Alpine 3.7 and switch to libressl --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index bec884ce9b..8085f3d354 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ -FROM python:2-alpine +FROM docker.io/python:2-alpine3.7 -RUN apk add --no-cache --virtual .nacl_deps su-exec build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers postgresql-dev +RUN apk add --no-cache --virtual .nacl_deps su-exec build-base libffi-dev zlib-dev libressl-dev libjpeg-turbo-dev linux-headers postgresql-dev COPY . /synapse From d8680c969bb2e8248436d7352ebf0f5cd1daea7b Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 21:55:35 +0100 Subject: [PATCH 036/310] Make it clear that the image has two modes of operation --- contrib/docker/README.md | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index b74c72698c..27e25afcef 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -21,23 +21,42 @@ You may have a local Python wheel cache available, in which case copy the releva ## Run +This image is designed to run either with an automatically generated configuration +file or with a custom configuration that requires manual edition. + +### Automated configuration + It is recommended that you use Docker Compose to run your containers, including this image and a Postgres server. A sample ``docker-compose.yml`` is provided, including example labels for reverse proxying and other artifacts. -Then, to run the server: +Read the section about environment variables and set at least mandatory variables, +then run the server: ``` docker-compose up -d ``` -In the case you specify a custom path for you configuration file and wish to -generate a fresh ``homeserver.yaml``, simply run: +### Manual configuration + +A sample ``docker-compose.yml`` is provided, including example labels for +reverse proxying and other artifacts. + +Specify a ``SYNAPSE_CONFIG_PATH``, preferably to a persistent path, +to use manual configuration. To generate a fresh ``homeserver.yaml``, simply run: ``` docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host synapse generate ``` +Then, customize your configuration and run the server: + +``` +docker-compose up -d +``` + +### Without Compose + If you do not wish to use Compose, you may still run this image using plain Docker commands. Note that the following is just a guideline and you may need to add parameters to the docker run command to account for the network situation @@ -49,10 +68,10 @@ docker run \ --name synapse \ -v ${DATA_PATH}:/data \ -e SYNAPSE_SERVER_NAME=my.matrix.host \ - matrixdotorg/synapse:latest + -e SYNAPSE_REPORT_STATS=yes \ + docker.io/matrixdotorg/synapse:latest ``` - ## Volumes The image expects a single volume, located at ``/data``, that will hold: From 48e2c641b80ac57d16701e4a27f348ea4f2c66cc Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 21:58:12 +0100 Subject: [PATCH 037/310] Specify the Docker registry in the build tag --- contrib/docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 27e25afcef..221d9c53b5 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -12,7 +12,7 @@ use that server. Build the docker image with the `docker build` command from the root of the synapse repository. ``` -docker build -t matrixdotorg/synapse . +docker build -t docker.io/matrixdotorg/synapse . ``` The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:` where `` is the same as the release tag in the synapse git repository. From a03c382966a3219acd26851db9cc6558e5c53310 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 22:00:43 +0100 Subject: [PATCH 038/310] Specify the Docker registry for the postgres image --- contrib/docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index e447bf1212..1d2aebbcd3 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -35,7 +35,7 @@ services: - traefik.port=8448 db: - image: postgres:10-alpine + image: docker.io/postgres:10-alpine # Change that password, of course! environment: - POSTGRES_USER=synapse From e511979fe6c4a03da3e9c1d16672e263f54ee2d3 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Fri, 9 Feb 2018 00:13:26 +0100 Subject: [PATCH 039/310] Make SYNAPSE_MACAROON_SECRET_KEY a mandatory option --- contrib/docker/README.md | 3 ++- contrib/docker/start.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 221d9c53b5..25c358c847 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -111,6 +111,8 @@ variables are available for configuration: * ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname. * ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous statistics reporting back to the Matrix project which helps us to get funding. +* ``SYNAPSE_MACAROON_SECRET_KEY`` (mandatory) secret for signing access tokens + to the server, set this to a proper random key. * ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if you run your own TLS-capable reverse proxy). * ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on @@ -130,7 +132,6 @@ Shared secrets, that will be initialized to random values if not set: * ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if registration is disable. -* ``SYNAPSE_MACAROON_SECRET_KEY``, secret for Macaroon. Database specific values (will use SQLite if not set): diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 32142bbe00..d4c1140b1d 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -44,8 +44,8 @@ else: if "SYNAPSE_CONFIG_PATH" in environ: args += ["--config-path", environ["SYNAPSE_CONFIG_PATH"]] else: - check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS")) - generate_secrets(environ, ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KEY")) + check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS", "SYNAPSE_MACAROON_SECRET_KEY")) + generate_secrets(environ, ("SYNAPSE_REGISTRATION_SHARED_SECRET",)) environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") if not os.path.exists("/compiled"): os.mkdir("/compiled") convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) From ca70148c0569295a2b9ecdd1cd9cd85a203f20e7 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Fri, 9 Feb 2018 00:23:19 +0100 Subject: [PATCH 040/310] Fix the path to the log config file --- contrib/docker/conf/homeserver.yaml | 2 +- contrib/docker/start.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 198b8ddee7..6bc25bb45f 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -69,7 +69,7 @@ database: event_cache_size: "{{ SYNAPSE_EVENT_CACHE_SIZE or "10K" }}" verbose: 0 log_file: "/data/homeserver.log" -log_config: "/data/{{ SYNAPSE_SERVER_NAME }}.log.config" +log_config: "/compiled/log.config" ## Ratelimiting ## diff --git a/contrib/docker/start.py b/contrib/docker/start.py index d4c1140b1d..75c30b8ac0 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -49,7 +49,7 @@ else: environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") if not os.path.exists("/compiled"): os.mkdir("/compiled") convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) - convert("/conf/log.config", "/compiled/%s.log.config" % environ["SYNAPSE_SERVER_NAME"], environ) + convert("/conf/log.config", "/compiled/log.config", environ) subprocess.check_output(["chown", "-R", ownership, "/data"]) args += ["--config-path", "/compiled/homeserver.yaml"] # Generate missing keys and start synapse From 6f0b1f85f9f34401219eab4b4977a63c698ce987 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sat, 10 Feb 2018 00:05:03 +0100 Subject: [PATCH 041/310] Generate macaroon and registration secrets, then store the results to the data dir --- contrib/docker/docker-compose.yml | 2 +- contrib/docker/start.py | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 1d2aebbcd3..9e32dd87de 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -6,7 +6,7 @@ version: '3' services: synapse: - image: docker.io/matrixdotorg/synapse:latest + image: synapse #docker.io/matrixdotorg/synapse:latest # Since snyapse does not retry to connect to the database, restart upon # failure restart: unless-stopped diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 75c30b8ac0..90e8b9c51a 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -16,10 +16,16 @@ def check_arguments(environ, args): sys.exit(2) def generate_secrets(environ, secrets): - for secret in secrets: + for name, secret in secrets.items(): if secret not in environ: - print("Generating a random secret for {}".format(secret)) - environ[secret] = os.urandom(32).encode("hex") + filename = "/data/%s.%s.key" % (environ["SYNAPSE_SERVER_NAME"], name) + if os.path.exists(filename): + with open(filename) as handle: value = handle.read() + else: + print("Generating a random secret for {}".format(name)) + value = os.urandom(32).encode("hex") + with open(filename, "w") as handle: handle.write(value) + environ[secret] = value # Prepare the configuration mode = sys.argv[1] if len(sys.argv) > 1 else None @@ -44,8 +50,11 @@ else: if "SYNAPSE_CONFIG_PATH" in environ: args += ["--config-path", environ["SYNAPSE_CONFIG_PATH"]] else: - check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS", "SYNAPSE_MACAROON_SECRET_KEY")) - generate_secrets(environ, ("SYNAPSE_REGISTRATION_SHARED_SECRET",)) + check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS")) + generate_secrets(environ, { + "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET", + "macaroon": "SYNAPSE_MACAROON_SECRET_KEY" + }) environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") if not os.path.exists("/compiled"): os.mkdir("/compiled") convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) From b815aa0e2db8f50116b1443b559ca13fe6ad1750 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sat, 10 Feb 2018 21:59:58 +0100 Subject: [PATCH 042/310] Remove an accidentally committed test configuration --- contrib/docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 9e32dd87de..1d2aebbcd3 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -6,7 +6,7 @@ version: '3' services: synapse: - image: synapse #docker.io/matrixdotorg/synapse:latest + image: docker.io/matrixdotorg/synapse:latest # Since snyapse does not retry to connect to the database, restart upon # failure restart: unless-stopped From 07f1b7181997dca91b67dca7561ce4c532caf253 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sat, 10 Feb 2018 23:57:36 +0100 Subject: [PATCH 043/310] Explicitely provide the postgres password to synapse in the Compose example --- contrib/docker/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 1d2aebbcd3..46e72601d3 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -15,6 +15,7 @@ services: - SYNAPSE_SERVER_NAME=my.matrix.host - SYNAPSE_REPORT_STATS=no - SYNAPSE_ENABLE_REGISTRATION=yes + - POSTGRES_PASSWORD=changeme volumes: # You may either store all the files in a local folder - ./files:/data From f44b7c022f6bc8b30cb8c446e0922b26b8b8eb5a Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sat, 10 Feb 2018 23:57:51 +0100 Subject: [PATCH 044/310] Disable logging to file and rely on the console when using Docker --- contrib/docker/conf/log.config | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/contrib/docker/conf/log.config b/contrib/docker/conf/log.config index 45e7eef953..b5c907c4f9 100644 --- a/contrib/docker/conf/log.config +++ b/contrib/docker/conf/log.config @@ -10,13 +10,6 @@ filters: request: "" handlers: - file: - class: logging.handlers.RotatingFileHandler - formatter: precise - filename: /data/homeserver.log - maxBytes: 104857600 - backupCount: 10 - filters: [context] console: class: logging.StreamHandler formatter: precise @@ -33,4 +26,4 @@ loggers: root: level: INFO - handlers: [file, console] + handlers: [console] From 5a6e54264d3a9f8b9b1f2e99db94e31ea3c21d24 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 9 Mar 2018 13:56:26 +0000 Subject: [PATCH 045/310] Make 'unexpected logging context' into warnings I think we've now fixed enough of these that the rest can be logged at warning. --- synapse/util/logcontext.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index d660ec785b..fdd4a93d07 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -163,7 +163,7 @@ class LoggingContext(object): current = self.set_current_context(self.previous_context) if current is not self: if current is self.sentinel: - logger.debug("Expected logging context %s has been lost", self) + logger.warn("Expected logging context %s has been lost", self) else: logger.warn( "Current logging context %s is not expected context %s", @@ -278,7 +278,7 @@ class PreserveLoggingContext(object): context = LoggingContext.set_current_context(self.current_context) if context != self.new_context: - logger.debug( + logger.warn( "Unexpected logging context: %s is not %s", context, self.new_context, ) From 7f8eebc8ee801467b0a04aa0fb07b1ba58866f22 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Sat, 7 Apr 2018 01:39:45 +0200 Subject: [PATCH 046/310] Open config file in non-bytes mode Nothing written into it is encoded, so it makes little sense, but it does break in python3 the way it was before. The variable names were adjusted to be less misleading. Signed-off-by: Adrian Tschira --- synapse/config/_base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 32b439d20a..b748ed2b0a 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -281,15 +281,15 @@ class Config(object): ) if not cls.path_exists(config_dir_path): os.makedirs(config_dir_path) - with open(config_path, "wb") as config_file: - config_bytes, config = obj.generate_config( + with open(config_path, "w") as config_file: + config_str, config = obj.generate_config( config_dir_path=config_dir_path, server_name=server_name, report_stats=(config_args.report_stats == "yes"), is_generating_file=True ) obj.invoke_all("generate_files", config) - config_file.write(config_bytes) + config_file.write(config_str) print(( "A config file has been generated in %r for server name" " %r with corresponding SSL keys and self-signed" From a3f9ddbeded33c61a715c86ba5029954ba22c22d Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Tue, 10 Apr 2018 17:22:52 +0200 Subject: [PATCH 047/310] Open certificate files as bytes That's what pyOpenSSL expects on python3 Signed-off-by: Adrian Tschira --- synapse/config/tls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 29eb012ddb..b66154bc7c 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -133,7 +133,7 @@ class TlsConfig(Config): tls_dh_params_path = config["tls_dh_params_path"] if not self.path_exists(tls_private_key_path): - with open(tls_private_key_path, "w") as private_key_file: + with open(tls_private_key_path, "wb") as private_key_file: tls_private_key = crypto.PKey() tls_private_key.generate_key(crypto.TYPE_RSA, 2048) private_key_pem = crypto.dump_privatekey( @@ -148,7 +148,7 @@ class TlsConfig(Config): ) if not self.path_exists(tls_certificate_path): - with open(tls_certificate_path, "w") as certificate_file: + with open(tls_certificate_path, "wb") as certificate_file: cert = crypto.X509() subject = cert.get_subject() subject.CN = config["server_name"] From bfc2ade9b3784469a5f7b80119fbd1679b6268c8 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Sun, 15 Apr 2018 17:24:25 +0200 Subject: [PATCH 048/310] Make event properties raise AttributeError instead They raised KeyError before. I'm changing this because the code uses hasattr() to check for the presence of a key. This worked accidentally before, because hasattr() silences all exceptions in python 2. However, in python3, this isn't the case anymore. I had a look around to see if anything depended on this raising a KeyError and I couldn't find anything. Of course, I could have simply missed it. Signed-off-by: Adrian Tschira --- synapse/events/__init__.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index e673e96cc0..d4d1b92f7a 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -48,13 +48,22 @@ class _EventInternalMetadata(object): def _event_dict_property(key): def getter(self): - return self._event_dict[key] + try: + return self._event_dict[key] + except KeyError: + raise AttributeError(key) def setter(self, v): - self._event_dict[key] = v + try: + self._event_dict[key] = v + except KeyError: + raise AttributeError(key) def delete(self): - del self._event_dict[key] + try: + del self._event_dict[key] + except KeyError: + raise AttributeError(key) return property( getter, From 2a3c33ff03aa88317c30da43cd3773c2789f0fcf Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Sun, 15 Apr 2018 17:15:16 +0200 Subject: [PATCH 049/310] Use six.moves.urlparse The imports were shuffled around a bunch in py3 Signed-off-by: Adrian Tschira --- synapse/config/appservice.py | 4 ++-- synapse/http/matrixfederationclient.py | 3 +-- synapse/rest/client/v1/login.py | 2 +- synapse/rest/client/v1/room.py | 9 +++++---- synapse/rest/media/v1/_base.py | 2 +- synapse/rest/media/v1/media_repository.py | 2 +- tests/rest/client/v1/test_rooms.py | 14 +++++++------- tests/utils.py | 5 ++--- 8 files changed, 20 insertions(+), 21 deletions(-) diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index 9a2359b6fd..277305e184 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -17,11 +17,11 @@ from ._base import Config, ConfigError from synapse.appservice import ApplicationService from synapse.types import UserID -import urllib import yaml import logging from six import string_types +from six.moves.urllib import parse as urlparse logger = logging.getLogger(__name__) @@ -105,7 +105,7 @@ def _load_appservice(hostname, as_info, config_filename): ) localpart = as_info["sender_localpart"] - if urllib.quote(localpart) != localpart: + if urlparse.quote(localpart) != localpart: raise ValueError( "sender_localpart needs characters which are not URL encoded." ) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 60a29081e8..c2e5610f50 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -38,8 +38,7 @@ import logging import random import sys import urllib -import urlparse - +from six.moves.urllib import parse as urlparse logger = logging.getLogger(__name__) outbound_logger = logging.getLogger("synapse.http.outbound") diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 45844aa2d2..34df5be4e9 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -25,7 +25,7 @@ from .base import ClientV1RestServlet, client_path_patterns import simplejson as json import urllib -import urlparse +from six.moves.urllib import parse as urlparse import logging from saml2 import BINDING_HTTP_POST diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 2ad0e5943b..fcf9c9ab44 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -28,8 +28,9 @@ from synapse.http.servlet import ( parse_json_object_from_request, parse_string, parse_integer ) +from six.moves.urllib import parse as urlparse + import logging -import urllib import simplejson as json logger = logging.getLogger(__name__) @@ -433,7 +434,7 @@ class RoomMessageListRestServlet(ClientV1RestServlet): as_client_event = "raw" not in request.args filter_bytes = request.args.get("filter", None) if filter_bytes: - filter_json = urllib.unquote(filter_bytes[-1]).decode("UTF-8") + filter_json = urlparse.unquote(filter_bytes[-1]).decode("UTF-8") event_filter = Filter(json.loads(filter_json)) else: event_filter = None @@ -718,8 +719,8 @@ class RoomTypingRestServlet(ClientV1RestServlet): def on_PUT(self, request, room_id, user_id): requester = yield self.auth.get_user_by_req(request) - room_id = urllib.unquote(room_id) - target_user = UserID.from_string(urllib.unquote(user_id)) + room_id = urlparse.unquote(room_id) + target_user = UserID.from_string(urlparse.unquote(user_id)) content = parse_json_object_from_request(request) diff --git a/synapse/rest/media/v1/_base.py b/synapse/rest/media/v1/_base.py index e7ac01da01..d9c4af9389 100644 --- a/synapse/rest/media/v1/_base.py +++ b/synapse/rest/media/v1/_base.py @@ -28,7 +28,7 @@ import os import logging import urllib -import urlparse +from six.moves.urllib import parse as urlparse logger = logging.getLogger(__name__) diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index bb79599379..9800ce7581 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -47,7 +47,7 @@ import shutil import cgi import logging -import urlparse +from six.moves.urllib import parse as urlparse logger = logging.getLogger(__name__) diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 7e8966a1a8..d763400eaf 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -24,7 +24,7 @@ from synapse.api.constants import Membership from synapse.types import UserID import json -import urllib +from six.moves.urllib import parse as urlparse from ....utils import MockHttpResource, setup_test_homeserver from .utils import RestTestCase @@ -766,7 +766,7 @@ class RoomMemberStateTestCase(RestTestCase): @defer.inlineCallbacks def test_rooms_members_self(self): path = "/rooms/%s/state/m.room.member/%s" % ( - urllib.quote(self.room_id), self.user_id + urlparse.quote(self.room_id), self.user_id ) # valid join message (NOOP since we made the room) @@ -786,7 +786,7 @@ class RoomMemberStateTestCase(RestTestCase): def test_rooms_members_other(self): self.other_id = "@zzsid1:red" path = "/rooms/%s/state/m.room.member/%s" % ( - urllib.quote(self.room_id), self.other_id + urlparse.quote(self.room_id), self.other_id ) # valid invite message @@ -802,7 +802,7 @@ class RoomMemberStateTestCase(RestTestCase): def test_rooms_members_other_custom_keys(self): self.other_id = "@zzsid1:red" path = "/rooms/%s/state/m.room.member/%s" % ( - urllib.quote(self.room_id), self.other_id + urlparse.quote(self.room_id), self.other_id ) # valid invite message with custom key @@ -859,7 +859,7 @@ class RoomMessagesTestCase(RestTestCase): @defer.inlineCallbacks def test_invalid_puts(self): path = "/rooms/%s/send/m.room.message/mid1" % ( - urllib.quote(self.room_id)) + urlparse.quote(self.room_id)) # missing keys or invalid json (code, response) = yield self.mock_resource.trigger( "PUT", path, '{}' @@ -894,7 +894,7 @@ class RoomMessagesTestCase(RestTestCase): @defer.inlineCallbacks def test_rooms_messages_sent(self): path = "/rooms/%s/send/m.room.message/mid1" % ( - urllib.quote(self.room_id)) + urlparse.quote(self.room_id)) content = '{"body":"test","msgtype":{"type":"a"}}' (code, response) = yield self.mock_resource.trigger("PUT", path, content) @@ -911,7 +911,7 @@ class RoomMessagesTestCase(RestTestCase): # m.text message type path = "/rooms/%s/send/m.room.message/mid2" % ( - urllib.quote(self.room_id)) + urlparse.quote(self.room_id)) content = '{"body":"test2","msgtype":"m.text"}' (code, response) = yield self.mock_resource.trigger("PUT", path, content) self.assertEquals(200, code, msg=str(response)) diff --git a/tests/utils.py b/tests/utils.py index f15317d27b..9e815d8643 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -15,8 +15,7 @@ import hashlib from inspect import getcallargs -import urllib -import urlparse +from six.moves.urllib import parse as urlparse from mock import Mock, patch from twisted.internet import defer, reactor @@ -234,7 +233,7 @@ class MockHttpResource(HttpServer): if matcher: try: args = [ - urllib.unquote(u).decode("UTF-8") + urlparse.unquote(u).decode("UTF-8") for u in matcher.groups() ] From 1ea904b9f09ea6a9df7792f61029d0250602d46b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 23 Apr 2018 00:53:18 +0100 Subject: [PATCH 050/310] Use deferred.addTimeout instead of time_bound_deferred This doesn't feel like a wheel we need to reinvent. --- synapse/http/__init__.py | 22 ++++++++++ synapse/http/client.py | 22 +++++----- synapse/http/matrixfederationclient.py | 35 ++++++++-------- synapse/notifier.py | 23 +++++------ synapse/util/__init__.py | 56 -------------------------- tests/util/test_clock.py | 33 --------------- 6 files changed, 60 insertions(+), 131 deletions(-) delete mode 100644 tests/util/test_clock.py diff --git a/synapse/http/__init__.py b/synapse/http/__init__.py index bfebb0f644..20e568bc43 100644 --- a/synapse/http/__init__.py +++ b/synapse/http/__init__.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,3 +13,24 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from twisted.internet.defer import CancelledError +from twisted.python import failure + +from synapse.api.errors import SynapseError + + +class RequestTimedOutError(SynapseError): + """Exception representing timeout of an outbound request""" + def __init__(self): + super(RequestTimedOutError, self).__init__(504, "Timed out") + + +def cancelled_to_request_timed_out_error(value): + """Turns CancelledErrors into RequestTimedOutErrors. + + For use with deferred.addTimeout() + """ + if isinstance(value, failure.Failure): + value.trap(CancelledError) + raise RequestTimedOutError() + return value diff --git a/synapse/http/client.py b/synapse/http/client.py index f3e4973c2e..35c8d51e71 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,9 +19,9 @@ from OpenSSL.SSL import VERIFY_NONE from synapse.api.errors import ( CodeMessageException, MatrixCodeMessageException, SynapseError, Codes, ) +from synapse.http import cancelled_to_request_timed_out_error from synapse.util.caches import CACHE_SIZE_FACTOR from synapse.util.logcontext import make_deferred_yieldable -from synapse.util import logcontext import synapse.metrics from synapse.http.endpoint import SpiderEndpoint @@ -95,21 +96,16 @@ class SimpleHttpClient(object): # counters to it outgoing_requests_counter.inc(method) - def send_request(): - request_deferred = self.agent.request( - method, uri, *args, **kwargs - ) - - return self.clock.time_bound_deferred( - request_deferred, - time_out=60, - ) - logger.info("Sending request %s %s", method, uri) try: - with logcontext.PreserveLoggingContext(): - response = yield send_request() + request_deferred = self.agent.request( + method, uri, *args, **kwargs + ) + request_deferred.addTimeout( + 60, reactor, cancelled_to_request_timed_out_error, + ) + response = yield make_deferred_yieldable(request_deferred) incoming_responses_counter.inc(method, response.code) logger.info( diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 60a29081e8..fe4b1636a6 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,17 +13,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import synapse.util.retryutils from twisted.internet import defer, reactor, protocol from twisted.internet.error import DNSLookupError from twisted.web.client import readBody, HTTPConnectionPool, Agent from twisted.web.http_headers import Headers from twisted.web._newclient import ResponseDone +from synapse.http import cancelled_to_request_timed_out_error from synapse.http.endpoint import matrix_federation_endpoint +import synapse.metrics from synapse.util.async import sleep from synapse.util import logcontext -import synapse.metrics +from synapse.util.logcontext import make_deferred_yieldable +import synapse.util.retryutils from canonicaljson import encode_canonical_json @@ -184,21 +187,19 @@ class MatrixFederationHttpClient(object): producer = body_callback(method, http_url_bytes, headers_dict) try: - def send_request(): - request_deferred = self.agent.request( - method, - url_bytes, - Headers(headers_dict), - producer - ) - - return self.clock.time_bound_deferred( - request_deferred, - time_out=timeout / 1000. if timeout else 60, - ) - - with logcontext.PreserveLoggingContext(): - response = yield send_request() + request_deferred = self.agent.request( + method, + url_bytes, + Headers(headers_dict), + producer + ) + request_deferred.addTimeout( + timeout / 1000. if timeout else 60, + reactor, cancelled_to_request_timed_out_error, + ) + response = yield make_deferred_yieldable( + request_deferred, + ) log_result = "%d %s" % (response.code, response.phrase,) break diff --git a/synapse/notifier.py b/synapse/notifier.py index 0e40a4aad6..1e4f78b993 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -13,12 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer +from twisted.internet import defer, reactor +from twisted.internet.defer import TimeoutError + from synapse.api.constants import EventTypes, Membership from synapse.api.errors import AuthError from synapse.handlers.presence import format_user_presence_state -from synapse.util import DeferredTimedOutError from synapse.util.logutils import log_function from synapse.util.async import ObservableDeferred from synapse.util.logcontext import PreserveLoggingContext, preserve_fn @@ -331,11 +332,11 @@ class Notifier(object): # Now we wait for the _NotifierUserStream to be told there # is a new token. listener = user_stream.new_listener(prev_token) + listener.deferred.addTimeout( + (end_time - now) / 1000., reactor, + ) with PreserveLoggingContext(): - yield self.clock.time_bound_deferred( - listener.deferred, - time_out=(end_time - now) / 1000. - ) + yield listener.deferred current_token = user_stream.current_token @@ -346,7 +347,7 @@ class Notifier(object): # Update the prev_token to the current_token since nothing # has happened between the old prev_token and the current_token prev_token = current_token - except DeferredTimedOutError: + except TimeoutError: break except defer.CancelledError: break @@ -551,13 +552,11 @@ class Notifier(object): if end_time <= now: break + listener.deferred.addTimeout((end_time - now) / 1000., reactor) try: with PreserveLoggingContext(): - yield self.clock.time_bound_deferred( - listener.deferred, - time_out=(end_time - now) / 1000. - ) - except DeferredTimedOutError: + yield listener.deferred + except TimeoutError: break except defer.CancelledError: break diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 756d8ffa32..814a7bf71b 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.api.errors import SynapseError from synapse.util.logcontext import PreserveLoggingContext from twisted.internet import defer, reactor, task @@ -24,11 +23,6 @@ import logging logger = logging.getLogger(__name__) -class DeferredTimedOutError(SynapseError): - def __init__(self): - super(DeferredTimedOutError, self).__init__(504, "Timed out") - - def unwrapFirstError(failure): # defer.gatherResults and DeferredLists wrap failures. failure.trap(defer.FirstError) @@ -85,53 +79,3 @@ class Clock(object): except Exception: if not ignore_errs: raise - - def time_bound_deferred(self, given_deferred, time_out): - if given_deferred.called: - return given_deferred - - ret_deferred = defer.Deferred() - - def timed_out_fn(): - e = DeferredTimedOutError() - - try: - ret_deferred.errback(e) - except Exception: - pass - - try: - given_deferred.cancel() - except Exception: - pass - - timer = None - - def cancel(res): - try: - self.cancel_call_later(timer) - except Exception: - pass - return res - - ret_deferred.addBoth(cancel) - - def success(res): - try: - ret_deferred.callback(res) - except Exception: - pass - - return res - - def err(res): - try: - ret_deferred.errback(res) - except Exception: - pass - - given_deferred.addCallbacks(callback=success, errback=err) - - timer = self.call_later(time_out, timed_out_fn) - - return ret_deferred diff --git a/tests/util/test_clock.py b/tests/util/test_clock.py deleted file mode 100644 index 9672603579..0000000000 --- a/tests/util/test_clock.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2017 Vector Creations Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from synapse import util -from twisted.internet import defer -from tests import unittest - - -class ClockTestCase(unittest.TestCase): - @defer.inlineCallbacks - def test_time_bound_deferred(self): - # just a deferred which never resolves - slow_deferred = defer.Deferred() - - clock = util.Clock() - time_bound = clock.time_bound_deferred(slow_deferred, 0.001) - - try: - yield time_bound - self.fail("Expected timedout error, but got nothing") - except util.DeferredTimedOutError: - pass From 48c01ae8511dee744879ac7207872770a0e24705 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 25 Apr 2018 11:02:28 +0100 Subject: [PATCH 051/310] ignore atom editor python ide files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c8901eb206..d438c6aeea 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,4 @@ env/ *.config .vscode/ +.ropeproject/ From e3a373f0023e99bdd2ff9bd37ded3571904b07cb Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 25 Apr 2018 14:58:43 +0100 Subject: [PATCH 052/310] remove duplicates from groups tables and rename inconsistently named indexes. Based on https://github.com/matrix-org/synapse/pull/3128 - thanks @vurpo\! --- .../schema/delta/48/group_unique_indexes.sql | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 synapse/storage/schema/delta/48/group_unique_indexes.sql diff --git a/synapse/storage/schema/delta/48/group_unique_indexes.sql b/synapse/storage/schema/delta/48/group_unique_indexes.sql new file mode 100644 index 0000000000..9ea7a8f2e6 --- /dev/null +++ b/synapse/storage/schema/delta/48/group_unique_indexes.sql @@ -0,0 +1,34 @@ +/* Copyright 2018 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- remove duplicates from group_users table +DELETE FROM group_users WHERE ctid NOT IN ( + SELECT min(ctid) FROM group_users GROUP BY group_id, user_id +); +DROP INDEX groups_users_g_idx; +CREATE UNIQUE INDEX group_users_g_idx ON group_users(group_id, user_id); + +-- remove duplicates from group_invites table +DELETE FROM group_invites WHERE ctid NOT IN ( + SELECT min(ctid) FROM group_invites GROUP BY group_id, user_id +); +DROP INDEX groups_invites_g_idx; +CREATE UNIQUE INDEX group_invites_g_idx ON group_invites(group_id, user_id); + +-- rename other indexes to actually match their table names... +ALTER INDEX groups_users_u_idx RENAME TO group_users_u_idx; +ALTER INDEX groups_invites_u_idx RENAME TO group_invites_u_idx; +ALTER INDEX groups_rooms_g_idx RENAME TO group_rooms_g_idx; +ALTER INDEX groups_rooms_r_idx RENAME TO group_rooms_r_idx; From 617bf40924bc8d627734eb2fb25a9f3980ede5d5 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 25 Apr 2018 17:37:29 +0100 Subject: [PATCH 053/310] Generate user daily stats --- synapse/app/homeserver.py | 6 ++ synapse/storage/__init__.py | 60 +++++++++++++++++-- synapse/storage/client_ips.py | 7 +++ synapse/storage/prepare_database.py | 2 +- .../schema/delta/49/add_user_daily_visits.sql | 25 ++++++++ .../49/add_user_ips_last_seen_only_index.sql | 17 ++++++ 6 files changed, 112 insertions(+), 5 deletions(-) create mode 100644 synapse/storage/schema/delta/49/add_user_daily_visits.sql create mode 100644 synapse/storage/schema/delta/49/add_user_ips_last_seen_only_index.sql diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index a0e465d644..808567365a 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -473,6 +473,9 @@ def run(hs): " changes across releases." ) + def generate_user_daily_visit_stats(): + hs.get_datastore().generate_user_daily_visits() + if hs.config.report_stats: logger.info("Scheduling stats reporting for 3 hour intervals") clock.looping_call(phone_stats_home, 3 * 60 * 60 * 1000) @@ -485,6 +488,9 @@ def run(hs): # be quite busy the first few minutes clock.call_later(5 * 60, phone_stats_home) + clock.looping_call(generate_user_daily_visit_stats, 60 * 1000) + clock.call_later(5 * 60, generate_user_daily_visit_stats) + if hs.config.daemonize and hs.config.print_pidfile: print (hs.config.pid_file) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 8cdfd50f90..6f2f947433 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -14,6 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import datetime +import time +import logging + from synapse.storage.devices import DeviceStore from .appservice import ( ApplicationServiceStore, ApplicationServiceTransactionStore @@ -55,10 +59,6 @@ from .engines import PostgresEngine from synapse.api.constants import PresenceState from synapse.util.caches.stream_change_cache import StreamChangeCache - -import logging - - logger = logging.getLogger(__name__) @@ -347,6 +347,58 @@ class DataStore(RoomMemberStore, RoomStore, return self.runInteraction("count_r30_users", _count_r30_users) + + def generate_user_daily_visits(self): + """ + Generates daily visit data for use in cohort/ retention analysis + """ + def _generate_user_daily_visits(txn): + logger.info("Calling _generate_user_daily_visits") + # determine timestamp of previous days + yesterday = datetime.datetime.now() - datetime.timedelta(days=1) + yesterday_start = datetime.datetime(yesterday.year, + yesterday.month, + yesterday.day, 0, 0, 0, 0) + yesterday_start_time = int(time.mktime(yesterday_start.timetuple())) * 1000 + + # Check that this job has not already been completed + sql = """ + SELECT timestamp + FROM user_daily_visits + ORDER by timestamp desc limit 1 + """ + txn.execute(sql) + row = txn.fetchone() + + # Bail if the most recent time is yesterday + if row and row[0] == yesterday_start_time: + logger.info("Bailing from _generate_user_daily_visits, already completed") + return + logger.info("inserting into user_daily_visits") + # Not specificying an upper bound means that if the update is run at + # 10 mins past midnight and the user is active during a 30 min session + # that the user is still included in the previous days stats + # This does mean that if the update is run hours late, then it is possible + # to overstate the cohort, but this seems a reasonable trade off + # The alternative is to insert on every request - but prefer to avoid + # for performance reasons + sql = """ + SELECT user_id, user_agent, device_id + FROM user_ips + WHERE last_seen > ? + """ + txn.execute(sql, (yesterday_start_time,)) + + sql = """ + INSERT INTO user_daily_visits (user_id, user_agent, device_id, timestamp) + VALUES (?, ?, ?, ?) + """ + + for row in txn: + txn.execute(sql, (row + (yesterday_start_time,))) + + return self.runInteraction("generate_user_daily_visits", _generate_user_daily_visits) + def get_users(self): """Function to reterive a list of users in users table. diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index 7b44dae0fc..ba46907737 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -55,6 +55,13 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): columns=["user_id", "last_seen"], ) + self.register_background_index_update( + "user_ips_last_seen_only_index", + index_name="user_ips_last_seen_only", + table="user_ips", + columns=["last_seen"], + ) + # (user_id, access_token, ip) -> (user_agent, device_id, last_seen) self._batch_row_update = {} diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index 04411a665f..c08e9cd65a 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 48 +SCHEMA_VERSION = 49 dir_path = os.path.abspath(os.path.dirname(__file__)) diff --git a/synapse/storage/schema/delta/49/add_user_daily_visits.sql b/synapse/storage/schema/delta/49/add_user_daily_visits.sql new file mode 100644 index 0000000000..11356a97fc --- /dev/null +++ b/synapse/storage/schema/delta/49/add_user_daily_visits.sql @@ -0,0 +1,25 @@ +/* Copyright 2018 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +CREATE TABLE user_daily_visits ( user_id TEXT NOT NULL, + device_id TEXT, + user_agent TEXT NOT NULL, + timestamp BIGINT NOT NULL ); + +/* What indexes should I include? + * Reads are offline so should optimise for writes + * Need to check if already an entry so user,day + */ diff --git a/synapse/storage/schema/delta/49/add_user_ips_last_seen_only_index.sql b/synapse/storage/schema/delta/49/add_user_ips_last_seen_only_index.sql new file mode 100644 index 0000000000..3a4ed59b5b --- /dev/null +++ b/synapse/storage/schema/delta/49/add_user_ips_last_seen_only_index.sql @@ -0,0 +1,17 @@ +/* Copyright 2018 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +INSERT into background_updates (update_name, progress_json) + VALUES ('user_ips_last_seen_only_index', '{}'); From fb6015d0a6828d57c18d601ea36b1a8a4cebc0e2 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 25 Apr 2018 17:56:11 +0100 Subject: [PATCH 054/310] pep8 --- synapse/storage/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 6f2f947433..c22d38800c 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -347,7 +347,6 @@ class DataStore(RoomMemberStore, RoomStore, return self.runInteraction("count_r30_users", _count_r30_users) - def generate_user_daily_visits(self): """ Generates daily visit data for use in cohort/ retention analysis @@ -390,14 +389,16 @@ class DataStore(RoomMemberStore, RoomStore, txn.execute(sql, (yesterday_start_time,)) sql = """ - INSERT INTO user_daily_visits (user_id, user_agent, device_id, timestamp) + INSERT INTO user_daily_visits (user_id. user_agent, + device_id, timestamp) VALUES (?, ?, ?, ?) """ for row in txn: txn.execute(sql, (row + (yesterday_start_time,))) - return self.runInteraction("generate_user_daily_visits", _generate_user_daily_visits) + return self.runInteraction("generate_user_daily_visits", + _generate_user_daily_visits) def get_users(self): """Function to reterive a list of users in users table. From d78ada31662e0aeeb5d13ddd16aabf432574fffd Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 26 Apr 2018 12:34:40 +0100 Subject: [PATCH 055/310] Miscellaneous fixes to python_dependencies * add some doc about wtf this thing does * pin Twisted to < 18.4 * add explicit dep on six (fixes #3089) --- synapse/python_dependencies.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 5cabf7dabe..711cbb6c50 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -1,5 +1,6 @@ # Copyright 2015, 2016 OpenMarket Ltd # Copyright 2017 Vector Creations Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,6 +19,18 @@ from distutils.version import LooseVersion logger = logging.getLogger(__name__) +# this dict maps from python package name to a list of modules we expect it to +# provide. +# +# the key is a "requirement specifier", as used as a parameter to `pip +# install`[1], or an `install_requires` argument to `setuptools.setup` [2]. +# +# the value is a sequence of strings; each entry should be the name of the +# python module, optionally followed by a version assertion which can be either +# ">=" or "==". +# +# [1] https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers. +# [2] https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-dependencies REQUIREMENTS = { "jsonschema>=2.5.1": ["jsonschema>=2.5.1"], "frozendict>=0.4": ["frozendict"], @@ -26,7 +39,11 @@ REQUIREMENTS = { "signedjson>=1.0.0": ["signedjson>=1.0.0"], "pynacl>=1.2.1": ["nacl>=1.2.1", "nacl.bindings"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], - "Twisted>=16.0.0": ["twisted>=16.0.0"], + + # we break under Twisted 18.4 + # (https://github.com/matrix-org/synapse/issues/3135) + "Twisted>=16.0.0,<18.4": ["twisted>=16.0.0"], + "pyopenssl>=0.14": ["OpenSSL>=0.14"], "pyyaml": ["yaml"], "pyasn1": ["pyasn1"], @@ -39,6 +56,7 @@ REQUIREMENTS = { "pymacaroons-pynacl": ["pymacaroons"], "msgpack-python>=0.3.0": ["msgpack"], "phonenumbers>=8.2.0": ["phonenumbers"], + "six": ["six"], } CONDITIONAL_REQUIREMENTS = { "web_client": { From dbf76fd4b9aa555ef9ab11455a9ed77014d991bd Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 26 Apr 2018 13:05:30 +0100 Subject: [PATCH 056/310] jenkins build: make sure we have a recent setuptools --- jenkins/prepare_synapse.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/jenkins/prepare_synapse.sh b/jenkins/prepare_synapse.sh index ffcb1cfab9..219828fa7b 100755 --- a/jenkins/prepare_synapse.sh +++ b/jenkins/prepare_synapse.sh @@ -1,5 +1,7 @@ #! /bin/bash +set -eux + cd "`dirname $0`/.." TOX_DIR=$WORKSPACE/.tox @@ -14,7 +16,16 @@ fi tox -e py27 --notest -v TOX_BIN=$TOX_DIR/py27/bin -$TOX_BIN/pip install setuptools + +# cryptography 2.2 requires setuptools >= 18.5. +# +# older versions of virtualenv (?) give us a virtualenv with the same version +# of setuptools as is installed on the system python (and tox runs virtualenv +# under python3, so we get the version of setuptools that is installed on that). +# +# anyway, make sure that we have a recent enough setuptools. +$TOX_BIN/pip install 'setuptools>=18.5' + { python synapse/python_dependencies.py echo lxml psycopg2 } | xargs $TOX_BIN/pip install From 31c8be956f9addc801808f9b22108326287f7143 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Apr 2018 01:56:58 +0100 Subject: [PATCH 057/310] also upgrade pip when installing --- jenkins/prepare_synapse.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jenkins/prepare_synapse.sh b/jenkins/prepare_synapse.sh index 219828fa7b..a30179f2aa 100755 --- a/jenkins/prepare_synapse.sh +++ b/jenkins/prepare_synapse.sh @@ -26,6 +26,10 @@ TOX_BIN=$TOX_DIR/py27/bin # anyway, make sure that we have a recent enough setuptools. $TOX_BIN/pip install 'setuptools>=18.5' +# we also need a semi-recent version of pip, because old ones fail to install +# the "enum34" dependency of cryptography. +$TOX_BIN/pip install 'pip>=10' + { python synapse/python_dependencies.py echo lxml psycopg2 } | xargs $TOX_BIN/pip install From 9255a6cb17716c022ebae1dbe9c142b78ca86ea7 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Apr 2018 11:07:40 +0100 Subject: [PATCH 058/310] Improve exception handling for background processes There were a bunch of places where we fire off a process to happen in the background, but don't have any exception handling on it - instead relying on the unhandled error being logged when the relevent deferred gets garbage-collected. This is unsatisfactory for a number of reasons: - logging on garbage collection is best-effort and may happen some time after the error, if at all - it can be hard to figure out where the error actually happened. - it is logged as a scary CRITICAL error which (a) I always forget to grep for and (b) it's not really CRITICAL if a background process we don't care about fails. So this is an attempt to add exception handling to everything we fire off into the background. --- synapse/app/appservice.py | 15 ++-- synapse/app/federation_sender.py | 25 +++--- synapse/app/pusher.py | 31 ++++---- synapse/app/synchrotron.py | 97 ++++++++++++----------- synapse/app/user_dir.py | 13 ++- synapse/appservice/scheduler.py | 25 +++--- synapse/crypto/keyring.py | 91 +++++++++++---------- synapse/federation/transaction_queue.py | 2 + synapse/federation/transport/server.py | 13 ++- synapse/groups/attestations.py | 44 +++++----- synapse/handlers/message.py | 22 +++-- synapse/handlers/presence.py | 19 ++++- synapse/handlers/receipts.py | 55 +++++++------ synapse/handlers/typing.py | 43 +++++----- synapse/notifier.py | 13 ++- synapse/push/emailpusher.py | 11 ++- synapse/push/httppusher.py | 5 +- synapse/rest/media/v1/storage_provider.py | 9 ++- synapse/storage/event_push_actions.py | 24 ++++-- synapse/util/logcontext.py | 7 +- 20 files changed, 331 insertions(+), 233 deletions(-) diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py index f2540023a7..58f2c9d68c 100644 --- a/synapse/app/appservice.py +++ b/synapse/app/appservice.py @@ -32,10 +32,10 @@ from synapse.replication.tcp.client import ReplicationClientHandler from synapse.server import HomeServer from synapse.storage.engines import create_engine from synapse.util.httpresourcetree import create_resource_tree -from synapse.util.logcontext import LoggingContext, preserve_fn +from synapse.util.logcontext import LoggingContext, run_in_background from synapse.util.manhole import manhole from synapse.util.versionstring import get_version_string -from twisted.internet import reactor +from twisted.internet import reactor, defer from twisted.web.resource import NoResource logger = logging.getLogger("synapse.app.appservice") @@ -112,9 +112,14 @@ class ASReplicationHandler(ReplicationClientHandler): if stream_name == "events": max_stream_id = self.store.get_room_max_stream_ordering() - preserve_fn( - self.appservice_handler.notify_interested_services - )(max_stream_id) + run_in_background(self._notify_app_services, max_stream_id) + + @defer.inlineCallbacks + def _notify_app_services(self, room_stream_id): + try: + yield self.appservice_handler.notify_interested_services(room_stream_id) + except Exception: + logger.exception("Error notifying application services of event") def start(config_options): diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py index 0cc3331519..4f2a9ca21a 100644 --- a/synapse/app/federation_sender.py +++ b/synapse/app/federation_sender.py @@ -237,19 +237,22 @@ class FederationSenderHandler(object): @defer.inlineCallbacks def update_token(self, token): - self.federation_position = token + try: + self.federation_position = token - # We linearize here to ensure we don't have races updating the token - with (yield self._fed_position_linearizer.queue(None)): - if self._last_ack < self.federation_position: - yield self.store.update_federation_out_pos( - "federation", self.federation_position - ) + # We linearize here to ensure we don't have races updating the token + with (yield self._fed_position_linearizer.queue(None)): + if self._last_ack < self.federation_position: + yield self.store.update_federation_out_pos( + "federation", self.federation_position + ) - # We ACK this token over replication so that the master can drop - # its in memory queues - self.replication_client.send_federation_ack(self.federation_position) - self._last_ack = self.federation_position + # We ACK this token over replication so that the master can drop + # its in memory queues + self.replication_client.send_federation_ack(self.federation_position) + self._last_ack = self.federation_position + except Exception: + logger.exception("Error updating federation stream position") if __name__ == '__main__': diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index d5c3a85195..739d113ad5 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -144,20 +144,23 @@ class PusherReplicationHandler(ReplicationClientHandler): @defer.inlineCallbacks def poke_pushers(self, stream_name, token, rows): - if stream_name == "pushers": - for row in rows: - if row.deleted: - yield self.stop_pusher(row.user_id, row.app_id, row.pushkey) - else: - yield self.start_pusher(row.user_id, row.app_id, row.pushkey) - elif stream_name == "events": - yield self.pusher_pool.on_new_notifications( - token, token, - ) - elif stream_name == "receipts": - yield self.pusher_pool.on_new_receipts( - token, token, set(row.room_id for row in rows) - ) + try: + if stream_name == "pushers": + for row in rows: + if row.deleted: + yield self.stop_pusher(row.user_id, row.app_id, row.pushkey) + else: + yield self.start_pusher(row.user_id, row.app_id, row.pushkey) + elif stream_name == "events": + yield self.pusher_pool.on_new_notifications( + token, token, + ) + elif stream_name == "receipts": + yield self.pusher_pool.on_new_receipts( + token, token, set(row.room_id for row in rows) + ) + except Exception: + logger.exception("Error poking pushers") def stop_pusher(self, user_id, app_id, pushkey): key = "%s:%s" % (app_id, pushkey) diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index 2fddcd935a..777da564d7 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -340,55 +340,58 @@ class SyncReplicationHandler(ReplicationClientHandler): @defer.inlineCallbacks def process_and_notify(self, stream_name, token, rows): - if stream_name == "events": - # We shouldn't get multiple rows per token for events stream, so - # we don't need to optimise this for multiple rows. - for row in rows: - event = yield self.store.get_event(row.event_id) - extra_users = () - if event.type == EventTypes.Member: - extra_users = (event.state_key,) - max_token = self.store.get_room_max_stream_ordering() - self.notifier.on_new_room_event( - event, token, max_token, extra_users - ) - elif stream_name == "push_rules": - self.notifier.on_new_event( - "push_rules_key", token, users=[row.user_id for row in rows], - ) - elif stream_name in ("account_data", "tag_account_data",): - self.notifier.on_new_event( - "account_data_key", token, users=[row.user_id for row in rows], - ) - elif stream_name == "receipts": - self.notifier.on_new_event( - "receipt_key", token, rooms=[row.room_id for row in rows], - ) - elif stream_name == "typing": - self.typing_handler.process_replication_rows(token, rows) - self.notifier.on_new_event( - "typing_key", token, rooms=[row.room_id for row in rows], - ) - elif stream_name == "to_device": - entities = [row.entity for row in rows if row.entity.startswith("@")] - if entities: + try: + if stream_name == "events": + # We shouldn't get multiple rows per token for events stream, so + # we don't need to optimise this for multiple rows. + for row in rows: + event = yield self.store.get_event(row.event_id) + extra_users = () + if event.type == EventTypes.Member: + extra_users = (event.state_key,) + max_token = self.store.get_room_max_stream_ordering() + self.notifier.on_new_room_event( + event, token, max_token, extra_users + ) + elif stream_name == "push_rules": self.notifier.on_new_event( - "to_device_key", token, users=entities, + "push_rules_key", token, users=[row.user_id for row in rows], ) - elif stream_name == "device_lists": - all_room_ids = set() - for row in rows: - room_ids = yield self.store.get_rooms_for_user(row.user_id) - all_room_ids.update(room_ids) - self.notifier.on_new_event( - "device_list_key", token, rooms=all_room_ids, - ) - elif stream_name == "presence": - yield self.presence_handler.process_replication_rows(token, rows) - elif stream_name == "receipts": - self.notifier.on_new_event( - "groups_key", token, users=[row.user_id for row in rows], - ) + elif stream_name in ("account_data", "tag_account_data",): + self.notifier.on_new_event( + "account_data_key", token, users=[row.user_id for row in rows], + ) + elif stream_name == "receipts": + self.notifier.on_new_event( + "receipt_key", token, rooms=[row.room_id for row in rows], + ) + elif stream_name == "typing": + self.typing_handler.process_replication_rows(token, rows) + self.notifier.on_new_event( + "typing_key", token, rooms=[row.room_id for row in rows], + ) + elif stream_name == "to_device": + entities = [row.entity for row in rows if row.entity.startswith("@")] + if entities: + self.notifier.on_new_event( + "to_device_key", token, users=entities, + ) + elif stream_name == "device_lists": + all_room_ids = set() + for row in rows: + room_ids = yield self.store.get_rooms_for_user(row.user_id) + all_room_ids.update(room_ids) + self.notifier.on_new_event( + "device_list_key", token, rooms=all_room_ids, + ) + elif stream_name == "presence": + yield self.presence_handler.process_replication_rows(token, rows) + elif stream_name == "receipts": + self.notifier.on_new_event( + "groups_key", token, users=[row.user_id for row in rows], + ) + except Exception: + logger.exception("Error processing replication") def start(config_options): diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py index 5f845e80d1..5ba7e9b416 100644 --- a/synapse/app/user_dir.py +++ b/synapse/app/user_dir.py @@ -39,10 +39,10 @@ from synapse.storage.engines import create_engine from synapse.storage.user_directory import UserDirectoryStore from synapse.util.caches.stream_change_cache import StreamChangeCache from synapse.util.httpresourcetree import create_resource_tree -from synapse.util.logcontext import LoggingContext, preserve_fn +from synapse.util.logcontext import LoggingContext, run_in_background from synapse.util.manhole import manhole from synapse.util.versionstring import get_version_string -from twisted.internet import reactor +from twisted.internet import reactor, defer from twisted.web.resource import NoResource logger = logging.getLogger("synapse.app.user_dir") @@ -164,7 +164,14 @@ class UserDirectoryReplicationHandler(ReplicationClientHandler): stream_name, token, rows ) if stream_name == "current_state_deltas": - preserve_fn(self.user_directory.notify_new_event)() + run_in_background(self._notify_directory) + + @defer.inlineCallbacks + def _notify_directory(self): + try: + yield self.user_directory.notify_new_event() + except Exception: + logger.exception("Error notifiying user directory of state update") def start(config_options): diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 6da315473d..dfc8d1b42e 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -176,17 +176,20 @@ class _TransactionController(object): @defer.inlineCallbacks def _start_recoverer(self, service): - yield self.store.set_appservice_state( - service, - ApplicationServiceState.DOWN - ) - logger.info( - "Application service falling behind. Starting recoverer. AS ID %s", - service.id - ) - recoverer = self.recoverer_fn(service, self.on_recovered) - self.add_recoverers([recoverer]) - recoverer.recover() + try: + yield self.store.set_appservice_state( + service, + ApplicationServiceState.DOWN + ) + logger.info( + "Application service falling behind. Starting recoverer. AS ID %s", + service.id + ) + recoverer = self.recoverer_fn(service, self.on_recovered) + self.add_recoverers([recoverer]) + recoverer.recover() + except Exception: + logger.exception("Error starting AS recoverer") @defer.inlineCallbacks def _is_service_up(self, service): diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index fce83d445f..32cbddbc53 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -146,53 +146,56 @@ class Keyring(object): verify_requests (List[VerifyKeyRequest]): """ - # create a deferred for each server we're going to look up the keys - # for; we'll resolve them once we have completed our lookups. - # These will be passed into wait_for_previous_lookups to block - # any other lookups until we have finished. - # The deferreds are called with no logcontext. - server_to_deferred = { - rq.server_name: defer.Deferred() - for rq in verify_requests - } + try: + # create a deferred for each server we're going to look up the keys + # for; we'll resolve them once we have completed our lookups. + # These will be passed into wait_for_previous_lookups to block + # any other lookups until we have finished. + # The deferreds are called with no logcontext. + server_to_deferred = { + rq.server_name: defer.Deferred() + for rq in verify_requests + } - # We want to wait for any previous lookups to complete before - # proceeding. - yield self.wait_for_previous_lookups( - [rq.server_name for rq in verify_requests], - server_to_deferred, - ) - - # Actually start fetching keys. - self._get_server_verify_keys(verify_requests) - - # When we've finished fetching all the keys for a given server_name, - # resolve the deferred passed to `wait_for_previous_lookups` so that - # any lookups waiting will proceed. - # - # map from server name to a set of request ids - server_to_request_ids = {} - - for verify_request in verify_requests: - server_name = verify_request.server_name - request_id = id(verify_request) - server_to_request_ids.setdefault(server_name, set()).add(request_id) - - def remove_deferreds(res, verify_request): - server_name = verify_request.server_name - request_id = id(verify_request) - server_to_request_ids[server_name].discard(request_id) - if not server_to_request_ids[server_name]: - d = server_to_deferred.pop(server_name, None) - if d: - d.callback(None) - return res - - for verify_request in verify_requests: - verify_request.deferred.addBoth( - remove_deferreds, verify_request, + # We want to wait for any previous lookups to complete before + # proceeding. + yield self.wait_for_previous_lookups( + [rq.server_name for rq in verify_requests], + server_to_deferred, ) + # Actually start fetching keys. + self._get_server_verify_keys(verify_requests) + + # When we've finished fetching all the keys for a given server_name, + # resolve the deferred passed to `wait_for_previous_lookups` so that + # any lookups waiting will proceed. + # + # map from server name to a set of request ids + server_to_request_ids = {} + + for verify_request in verify_requests: + server_name = verify_request.server_name + request_id = id(verify_request) + server_to_request_ids.setdefault(server_name, set()).add(request_id) + + def remove_deferreds(res, verify_request): + server_name = verify_request.server_name + request_id = id(verify_request) + server_to_request_ids[server_name].discard(request_id) + if not server_to_request_ids[server_name]: + d = server_to_deferred.pop(server_name, None) + if d: + d.callback(None) + return res + + for verify_request in verify_requests: + verify_request.deferred.addBoth( + remove_deferreds, verify_request, + ) + except Exception: + logger.exception("Error starting key lookups") + @defer.inlineCallbacks def wait_for_previous_lookups(self, server_names, server_to_deferred): """Waits for any previous key lookups for the given servers to finish. diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 963d938edd..ded2b1871a 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -323,6 +323,8 @@ class TransactionQueue(object): break yield self._process_presence_inner(states_map.values()) + except Exception: + logger.exception("Error sending presence states to servers") finally: self._processing_pending_presence = False diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index ff0656df3e..19d09f5422 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -25,7 +25,7 @@ from synapse.http.servlet import ( ) from synapse.util.ratelimitutils import FederationRateLimiter from synapse.util.versionstring import get_version_string -from synapse.util.logcontext import preserve_fn +from synapse.util.logcontext import run_in_background from synapse.types import ThirdPartyInstanceID, get_domain_from_id import functools @@ -152,11 +152,18 @@ class Authenticator(object): # alive retry_timings = yield self.store.get_destination_retry_timings(origin) if retry_timings and retry_timings["retry_last_ts"]: - logger.info("Marking origin %r as up", origin) - preserve_fn(self.store.set_destination_retry_timings)(origin, 0, 0) + run_in_background(self._reset_retry_timings, origin) defer.returnValue(origin) + @defer.inlineCallbacks + def _reset_retry_timings(self, origin): + try: + logger.info("Marking origin %r as up", origin) + yield self.store.set_destination_retry_timings(origin, 0, 0) + except Exception: + logger.exception("Error resetting retry timings on %s", origin) + class BaseFederationServlet(object): REQUIRE_AUTH = True diff --git a/synapse/groups/attestations.py b/synapse/groups/attestations.py index 1fb709e6c3..7187df2508 100644 --- a/synapse/groups/attestations.py +++ b/synapse/groups/attestations.py @@ -165,28 +165,32 @@ class GroupAttestionRenewer(object): @defer.inlineCallbacks def _renew_attestation(group_id, user_id): - if not self.is_mine_id(group_id): - destination = get_domain_from_id(group_id) - elif not self.is_mine_id(user_id): - destination = get_domain_from_id(user_id) - else: - logger.warn( - "Incorrectly trying to do attestations for user: %r in %r", - user_id, group_id, + try: + if not self.is_mine_id(group_id): + destination = get_domain_from_id(group_id) + elif not self.is_mine_id(user_id): + destination = get_domain_from_id(user_id) + else: + logger.warn( + "Incorrectly trying to do attestations for user: %r in %r", + user_id, group_id, + ) + yield self.store.remove_attestation_renewal(group_id, user_id) + return + + attestation = self.attestations.create_attestation(group_id, user_id) + + yield self.transport_client.renew_group_attestation( + destination, group_id, user_id, + content={"attestation": attestation}, ) - yield self.store.remove_attestation_renewal(group_id, user_id) - return - attestation = self.attestations.create_attestation(group_id, user_id) - - yield self.transport_client.renew_group_attestation( - destination, group_id, user_id, - content={"attestation": attestation}, - ) - - yield self.store.update_attestation_renewal( - group_id, user_id, attestation - ) + yield self.store.update_attestation_renewal( + group_id, user_id, attestation + ) + except Exception: + logger.exception("Error renewing attestation of %r in %r", + user_id, group_id) for row in rows: group_id = row["group_id"] diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 21628a8540..d168ff5b86 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -857,15 +857,25 @@ class EventCreationHandler(object): @defer.inlineCallbacks def _notify(): yield run_on_reactor() - self.notifier.on_new_room_event( - event, event_stream_id, max_stream_id, - extra_users=extra_users - ) + try: + self.notifier.on_new_room_event( + event, event_stream_id, max_stream_id, + extra_users=extra_users + ) + except Exception: + logger.exception("Error notifying about new room event") preserve_fn(_notify)() if event.type == EventTypes.Message: - presence = self.hs.get_presence_handler() # We don't want to block sending messages on any presence code. This # matters as sometimes presence code can take a while. - preserve_fn(presence.bump_presence_active_time)(requester.user) + run_in_background(self._bump_active_time, requester.user) + + @defer.inlineCallbacks + def _bump_active_time(self, user): + try: + presence = self.hs.get_presence_handler() + yield presence.bump_presence_active_time(user) + except Exception: + logger.exception("Error bumping presence active time") diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index a5e501897c..585f3e4da2 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -31,7 +31,7 @@ from synapse.storage.presence import UserPresenceState from synapse.util.caches.descriptors import cachedInlineCallbacks from synapse.util.async import Linearizer -from synapse.util.logcontext import preserve_fn +from synapse.util.logcontext import run_in_background from synapse.util.logutils import log_function from synapse.util.metrics import Measure from synapse.util.wheel_timer import WheelTimer @@ -254,6 +254,14 @@ class PresenceHandler(object): logger.info("Finished _persist_unpersisted_changes") + @defer.inlineCallbacks + def _update_states_and_catch_exception(self, new_states): + try: + res = yield self._update_states(new_states) + defer.returnValue(res) + except Exception: + logger.exception("Error updating presence") + @defer.inlineCallbacks def _update_states(self, new_states): """Updates presence of users. Sets the appropriate timeouts. Pokes @@ -364,7 +372,7 @@ class PresenceHandler(object): now=now, ) - preserve_fn(self._update_states)(changes) + run_in_background(self._update_states_and_catch_exception, changes) except Exception: logger.exception("Exception in _handle_timeouts loop") @@ -422,20 +430,23 @@ class PresenceHandler(object): @defer.inlineCallbacks def _end(): - if affect_presence: + try: self.user_to_num_current_syncs[user_id] -= 1 prev_state = yield self.current_state_for_user(user_id) yield self._update_states([prev_state.copy_and_replace( last_user_sync_ts=self.clock.time_msec(), )]) + except Exception: + logger.exception("Error updating presence after sync") @contextmanager def _user_syncing(): try: yield finally: - preserve_fn(_end)() + if affect_presence: + run_in_background(_end) defer.returnValue(_user_syncing()) diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index 3f215c2b4e..2e0672161c 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -135,37 +135,40 @@ class ReceiptsHandler(BaseHandler): """Given a list of receipts, works out which remote servers should be poked and pokes them. """ - # TODO: Some of this stuff should be coallesced. - for receipt in receipts: - room_id = receipt["room_id"] - receipt_type = receipt["receipt_type"] - user_id = receipt["user_id"] - event_ids = receipt["event_ids"] - data = receipt["data"] + try: + # TODO: Some of this stuff should be coallesced. + for receipt in receipts: + room_id = receipt["room_id"] + receipt_type = receipt["receipt_type"] + user_id = receipt["user_id"] + event_ids = receipt["event_ids"] + data = receipt["data"] - users = yield self.state.get_current_user_in_room(room_id) - remotedomains = set(get_domain_from_id(u) for u in users) - remotedomains = remotedomains.copy() - remotedomains.discard(self.server_name) + users = yield self.state.get_current_user_in_room(room_id) + remotedomains = set(get_domain_from_id(u) for u in users) + remotedomains = remotedomains.copy() + remotedomains.discard(self.server_name) - logger.debug("Sending receipt to: %r", remotedomains) + logger.debug("Sending receipt to: %r", remotedomains) - for domain in remotedomains: - self.federation.send_edu( - destination=domain, - edu_type="m.receipt", - content={ - room_id: { - receipt_type: { - user_id: { - "event_ids": event_ids, - "data": data, + for domain in remotedomains: + self.federation.send_edu( + destination=domain, + edu_type="m.receipt", + content={ + room_id: { + receipt_type: { + user_id: { + "event_ids": event_ids, + "data": data, + } } - } + }, }, - }, - key=(room_id, receipt_type, user_id), - ) + key=(room_id, receipt_type, user_id), + ) + except Exception: + logger.exception("Error pushing receipts to remote servers") @defer.inlineCallbacks def get_receipts_for_room(self, room_id, to_key): diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 77c0cf146f..823e2e27e1 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -205,28 +205,31 @@ class TypingHandler(object): @defer.inlineCallbacks def _push_remote(self, member, typing): - users = yield self.state.get_current_user_in_room(member.room_id) - self._member_last_federation_poke[member] = self.clock.time_msec() + try: + users = yield self.state.get_current_user_in_room(member.room_id) + self._member_last_federation_poke[member] = self.clock.time_msec() - now = self.clock.time_msec() - self.wheel_timer.insert( - now=now, - obj=member, - then=now + FEDERATION_PING_INTERVAL, - ) + now = self.clock.time_msec() + self.wheel_timer.insert( + now=now, + obj=member, + then=now + FEDERATION_PING_INTERVAL, + ) - for domain in set(get_domain_from_id(u) for u in users): - if domain != self.server_name: - self.federation.send_edu( - destination=domain, - edu_type="m.typing", - content={ - "room_id": member.room_id, - "user_id": member.user_id, - "typing": typing, - }, - key=member, - ) + for domain in set(get_domain_from_id(u) for u in users): + if domain != self.server_name: + self.federation.send_edu( + destination=domain, + edu_type="m.typing", + content={ + "room_id": member.room_id, + "user_id": member.user_id, + "typing": typing, + }, + key=member, + ) + except Exception: + logger.exception("Error pushing typing notif to remotes") @defer.inlineCallbacks def _recv_edu(self, origin, content): diff --git a/synapse/notifier.py b/synapse/notifier.py index 0e40a4aad6..939723a404 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -21,7 +21,7 @@ from synapse.handlers.presence import format_user_presence_state from synapse.util import DeferredTimedOutError from synapse.util.logutils import log_function from synapse.util.async import ObservableDeferred -from synapse.util.logcontext import PreserveLoggingContext, preserve_fn +from synapse.util.logcontext import PreserveLoggingContext, run_in_background from synapse.util.metrics import Measure from synapse.types import StreamToken from synapse.visibility import filter_events_for_client @@ -251,9 +251,7 @@ class Notifier(object): def _on_new_room_event(self, event, room_stream_id, extra_users=[]): """Notify any user streams that are interested in this room event""" # poke any interested application service. - preserve_fn(self.appservice_handler.notify_interested_services)( - room_stream_id - ) + run_in_background(self._notify_app_services, room_stream_id) if self.federation_sender: self.federation_sender.notify_new_events(room_stream_id) @@ -267,6 +265,13 @@ class Notifier(object): rooms=[event.room_id], ) + @defer.inlineCallbacks + def _notify_app_services(self, room_stream_id): + try: + yield self.appservice_handler.notify_interested_services(room_stream_id) + except Exception: + logger.exception("Error notifying application services of event") + def on_new_event(self, stream_key, new_token, users=[], rooms=[]): """ Used to inform listeners that something has happend event wise. diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index 58df98a793..ba7286cb72 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -77,10 +77,13 @@ class EmailPusher(object): @defer.inlineCallbacks def on_started(self): if self.mailer is not None: - self.throttle_params = yield self.store.get_throttle_params_by_room( - self.pusher_id - ) - yield self._process() + try: + self.throttle_params = yield self.store.get_throttle_params_by_room( + self.pusher_id + ) + yield self._process() + except Exception: + logger.exception("Error starting email pusher") def on_stop(self): if self.timed_call: diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 2cbac571b8..1420d378ef 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -94,7 +94,10 @@ class HttpPusher(object): @defer.inlineCallbacks def on_started(self): - yield self._process() + try: + yield self._process() + except Exception: + logger.exception("Error starting http pusher") @defer.inlineCallbacks def on_new_notifications(self, min_stream_ordering, max_stream_ordering): diff --git a/synapse/rest/media/v1/storage_provider.py b/synapse/rest/media/v1/storage_provider.py index c188192f2b..0252afd9d3 100644 --- a/synapse/rest/media/v1/storage_provider.py +++ b/synapse/rest/media/v1/storage_provider.py @@ -18,7 +18,7 @@ from twisted.internet import defer, threads from .media_storage import FileResponder from synapse.config._base import Config -from synapse.util.logcontext import preserve_fn +from synapse.util.logcontext import run_in_background import logging import os @@ -87,7 +87,12 @@ class StorageProviderWrapper(StorageProvider): return self.backend.store_file(path, file_info) else: # TODO: Handle errors. - preserve_fn(self.backend.store_file)(path, file_info) + def store(): + try: + return self.backend.store_file(path, file_info) + except Exception: + logger.exception("Error storing file") + run_in_background(store) return defer.succeed(None) def fetch(self, path, file_info): diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index e78f8d0114..c22762eb5c 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -448,6 +448,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): "add_push_actions_to_staging", _add_push_actions_to_staging_txn ) + @defer.inlineCallbacks def remove_push_actions_from_staging(self, event_id): """Called if we failed to persist the event to ensure that stale push actions don't build up in the DB @@ -456,13 +457,22 @@ class EventPushActionsWorkerStore(SQLBaseStore): event_id (str) """ - return self._simple_delete( - table="event_push_actions_staging", - keyvalues={ - "event_id": event_id, - }, - desc="remove_push_actions_from_staging", - ) + try: + res = yield self._simple_delete( + table="event_push_actions_staging", + keyvalues={ + "event_id": event_id, + }, + desc="remove_push_actions_from_staging", + ) + defer.returnValue(res) + except Exception: + # this method is called from an exception handler, so propagating + # another exception here really isn't helpful - there's nothing + # the caller can do about it. Just log the exception and move on. + logger.exception( + "Error removing push actions after event persistence failure", + ) @defer.inlineCallbacks def _find_stream_orderings_for_times(self): diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index d59adc236e..d6587e4409 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -305,7 +305,12 @@ def run_in_background(f, *args, **kwargs): deferred returned by the funtion completes. Useful for wrapping functions that return a deferred which you don't yield - on. + on (for instance because you want to pass it to deferred.gatherResults()). + + Note that if you completely discard the result, you should make sure that + `f` doesn't raise any deferred exceptions, otherwise a scary-looking + CRITICAL error about an unhandled error will be logged without much + indication about where it came from. """ current = LoggingContext.current_context() res = f(*args, **kwargs) From 605defb9e4c274d61d9da86546e6fd6c78f3f6cf Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Apr 2018 11:16:28 +0100 Subject: [PATCH 059/310] Add missing consumeErrors In general we want defer.gatherResults to consumeErrors, rather than having exceptions hanging around and getting logged as CRITICAL unhandled errors. --- synapse/handlers/e2e_keys.py | 4 ++-- synapse/push/pusherpool.py | 8 ++++++-- synapse/storage/stream.py | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index 325c0c4a9f..7eb03ad32e 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -141,7 +141,7 @@ class E2eKeysHandler(object): yield make_deferred_yieldable(defer.gatherResults([ preserve_fn(do_remote_query)(destination) for destination in remote_queries_not_in_cache - ])) + ], consumeErrors=True)) defer.returnValue({ "device_keys": results, "failures": failures, @@ -244,7 +244,7 @@ class E2eKeysHandler(object): yield make_deferred_yieldable(defer.gatherResults([ preserve_fn(claim_client_keys)(destination) for destination in remote_queries - ])) + ], consumeErrors=True)) logger.info( "Claimed one-time-keys: %s", diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 134e89b371..2f467d1f9c 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -142,7 +142,9 @@ class PusherPool: ) ) - yield make_deferred_yieldable(defer.gatherResults(deferreds)) + yield make_deferred_yieldable( + defer.gatherResults(deferreds, consumeErrors=True), + ) except Exception: logger.exception("Exception in pusher on_new_notifications") @@ -167,7 +169,9 @@ class PusherPool: preserve_fn(p.on_new_receipts)(min_stream_id, max_stream_id) ) - yield make_deferred_yieldable(defer.gatherResults(deferreds)) + yield make_deferred_yieldable( + defer.gatherResults(deferreds, consumeErrors=True), + ) except Exception: logger.exception("Exception in pusher on_new_receipts") diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 2956c3b3e0..3b8b539993 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -202,7 +202,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): room_id, from_key, to_key, limit, order=order, ) for room_id in rm_ids - ])) + ], consumeErrors=True)) results.update(dict(zip(rm_ids, res))) defer.returnValue(results) From 6e10eed28e5da3700b6f545ad122dad1407cfb2c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 16 Feb 2018 11:16:29 +0000 Subject: [PATCH 060/310] Refactor event storage to not require state This is in preparation for using contexts that may or may not have the current_state_ids set. This will allow us to avoid unnecessarily pulling out state for an event on the master process when using workers. We also add a check to see if the state groups of the old extremities are the same as the new ones. --- synapse/storage/events.py | 104 ++++++++++++++++++++++++-------------- 1 file changed, 67 insertions(+), 37 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 5fe4a0e56c..a9269707df 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -22,7 +22,6 @@ import logging import simplejson as json from twisted.internet import defer - from synapse.storage.events_worker import EventsWorkerStore from synapse.util.async import ObservableDeferred from synapse.util.frozenutils import frozendict_json_encoder @@ -425,7 +424,9 @@ class EventsStore(EventsWorkerStore): ) current_state = yield self._get_new_state_after_events( room_id, - ev_ctx_rm, new_latest_event_ids, + ev_ctx_rm, + latest_event_ids, + new_latest_event_ids, ) if current_state is not None: current_state_for_room[room_id] = current_state @@ -513,7 +514,8 @@ class EventsStore(EventsWorkerStore): defer.returnValue(new_latest_event_ids) @defer.inlineCallbacks - def _get_new_state_after_events(self, room_id, events_context, new_latest_event_ids): + def _get_new_state_after_events(self, room_id, events_context, old_latest_event_ids, + new_latest_event_ids): """Calculate the current state dict after adding some new events to a room @@ -524,6 +526,9 @@ class EventsStore(EventsWorkerStore): events_context (list[(EventBase, EventContext)]): events and contexts which are being added to the room + old_latest_event_ids (iterable[str]): + the old forward extremities for the room. + new_latest_event_ids (iterable[str]): the new forward extremities for the room. @@ -534,64 +539,89 @@ class EventsStore(EventsWorkerStore): """ if not new_latest_event_ids: - defer.returnValue({}) + return # map from state_group to ((type, key) -> event_id) state map - state_groups = {} - missing_event_ids = [] - was_updated = False + state_groups_map = {} + for ev, ctx in events_context: + if ctx.state_group is None: + # I don't think this can happen, but let's double-check + raise Exception( + "Context for new extremity event %s has no state " + "group" % (ev.event_id, ), + ) + + if ctx.state_group in state_groups_map: + continue + + state_groups_map[ctx.state_group] = ctx.current_state_ids + + # We need to map the event_ids to their state groups. First, lets + # check if the event is one we're persisting and then we can pull the + # state group from its context. + # Otherwise we need to pull the state group from the database. + + # Set of events we need to fetch groups for. (We know none of the old + # extremities are going to be in events_context). + missing_event_ids = set(old_latest_event_ids) + + event_id_to_state_group = {} for event_id in new_latest_event_ids: - # First search in the list of new events we're adding, - # and then use the current state from that + # First search in the list of new events we're adding. for ev, ctx in events_context: if event_id == ev.event_id: - if ctx.current_state_ids is None: - raise Exception("Unknown current state") - - if ctx.state_group is None: - # I don't think this can happen, but let's double-check - raise Exception( - "Context for new extremity event %s has no state " - "group" % (event_id, ), - ) - - # If we've already seen the state group don't bother adding - # it to the state sets again - if ctx.state_group not in state_groups: - state_groups[ctx.state_group] = ctx.current_state_ids - if ctx.delta_ids or hasattr(ev, "state_key"): - was_updated = True + event_id_to_state_group[event_id] = ctx.state_group break else: # If we couldn't find it, then we'll need to pull # the state from the database - was_updated = True - missing_event_ids.append(event_id) - - if not was_updated: - return + missing_event_ids.add(event_id) if missing_event_ids: - # Now pull out the state for any missing events from DB + # Now pull out the state groups for any missing events from DB event_to_groups = yield self._get_state_group_for_events( missing_event_ids, ) + event_id_to_state_group.update(event_to_groups) - groups = set(event_to_groups.itervalues()) - set(state_groups.iterkeys()) + # State groups of old_latest_event_ids + old_state_groups = set( + event_id_to_state_group[evid] for evid in old_latest_event_ids + ) - if groups: - group_to_state = yield self._get_state_for_groups(groups) - state_groups.update(group_to_state) + # State groups of new_latest_event_ids + new_state_groups = set( + event_id_to_state_group[evid] for evid in new_latest_event_ids + ) - if len(state_groups) == 1: + # If they old and new groups are the same then we don't need to do + # anything. + if old_state_groups == new_state_groups: + return + + # Now that we have calculated new_state_groups we need to get + # their state IDs so we can resolve to a single state set. + missing_state = new_state_groups - set(state_groups_map) + if missing_state: + group_to_state = yield self._get_state_for_groups(missing_state) + state_groups_map.update(group_to_state) + + if len(new_state_groups) == 1: # If there is only one state group, then we know what the current # state is. - defer.returnValue(state_groups.values()[0]) + defer.returnValue(state_groups_map[new_state_groups.pop()]) + + # Ok, we need to defer to the state handler to resolve our state sets. def get_events(ev_ids): return self.get_events( ev_ids, get_prev_content=False, check_redacted=False, ) + + state_groups = { + sg: state_groups_map[sg] for sg in new_state_groups + } + events_map = {ev.event_id: ev for ev, _ in events_context} logger.debug("calling resolve_state_groups from preserve_events") res = yield self._state_resolution_handler.resolve_state_groups( From 6493b22b42685a6cc06c1113196d305e4d52deed Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Apr 2018 11:40:06 +0100 Subject: [PATCH 061/310] reraise exceptions more carefully We need to be careful (under python 2, at least) that when we reraise an exception after doing some error handling, we actually reraise the original exception rather than anything that might have been raised (and handled) during the error handling. --- synapse/handlers/federation.py | 16 ++++++++++------ synapse/handlers/message.py | 21 ++++++++++++++------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index ae7e0d6da2..260df025f9 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -19,9 +19,11 @@ import httplib import itertools import logging +import sys from signedjson.key import decode_verify_key_bytes from signedjson.sign import verify_signed_json +import six from twisted.internet import defer from unpaddedbase64 import decode_base64 @@ -1513,12 +1515,14 @@ class FederationHandler(BaseHandler): backfilled=backfilled, ) except: # noqa: E722, as we reraise the exception this is fine. - # Ensure that we actually remove the entries in the push actions - # staging area - logcontext.preserve_fn( - self.store.remove_push_actions_from_staging - )(event.event_id) - raise + tp, value, tb = sys.exc_info() + + logcontext.run_in_background( + self.store.remove_push_actions_from_staging, + event.event_id, + ) + + six.reraise(tp, value, tb) if not backfilled: # this intentionally does not yield: we don't care about the result diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 21628a8540..8e2e44bdcd 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -13,6 +13,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import logging +import simplejson +import sys + +from canonicaljson import encode_canonical_json +import six from twisted.internet import defer, reactor from twisted.python.failure import Failure @@ -34,11 +40,6 @@ from synapse.replication.http.send_event import send_event_to_master from ._base import BaseHandler -from canonicaljson import encode_canonical_json - -import logging -import simplejson - logger = logging.getLogger(__name__) @@ -729,8 +730,14 @@ class EventCreationHandler(object): except: # noqa: E722, as we reraise the exception this is fine. # Ensure that we actually remove the entries in the push actions # staging area, if we calculated them. - preserve_fn(self.store.remove_push_actions_from_staging)(event.event_id) - raise + tp, value, tb = sys.exc_info() + + run_in_background( + self.store.remove_push_actions_from_staging, + event.event_id, + ) + + six.reraise(tp, value, tb) @defer.inlineCallbacks def persist_and_notify_client_event( From 05ba7e3a44a689c67b67c617a9a23ce44da5a15e Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Fri, 27 Apr 2018 12:13:12 +0100 Subject: [PATCH 062/310] Update CHANGES.rst --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0c8307e439..40d13c6484 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,4 +1,4 @@ -Changes in synapse v0.28.0-rc1 (2018-04-26) +Changes in synapse v0.28.0 (2018-04-26) =========================================== Bug Fixes: From 13843f771ea64cfd4fb7e2f03854b9506f63fcfd Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Apr 2018 12:17:13 +0100 Subject: [PATCH 063/310] Trap exceptions thrown within run_in_background Turn any exceptions that get thrown synchronously within run_in_background into Failures instead. --- synapse/util/logcontext.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index d59adc236e..127581a0b4 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -308,7 +308,13 @@ def run_in_background(f, *args, **kwargs): on. """ current = LoggingContext.current_context() - res = f(*args, **kwargs) + try: + res = f(*args, **kwargs) + except: # noqa: E722 + # the assumption here is that the caller doesn't want to be disturbed + # by synchronous exceptions, so let's turn them into Failures. + return defer.fail() + if isinstance(res, defer.Deferred) and not res.called: # The function will have reset the context before returning, so # we need to restore it now. From 9d2c1b8429996cb9766ac636034485e2a1d685cc Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Apr 2018 12:52:30 +0100 Subject: [PATCH 064/310] Backport deferred.addTimeout Twisted 16.0 doesn't have addTimeout, so let's backport it. --- synapse/http/__init__.py | 2 +- synapse/http/client.py | 6 ++- synapse/http/matrixfederationclient.py | 7 +-- synapse/notifier.py | 22 ++++++--- synapse/util/async.py | 67 ++++++++++++++++++++++++++ 5 files changed, 90 insertions(+), 14 deletions(-) diff --git a/synapse/http/__init__.py b/synapse/http/__init__.py index 20e568bc43..0d47ccdb59 100644 --- a/synapse/http/__init__.py +++ b/synapse/http/__init__.py @@ -28,7 +28,7 @@ class RequestTimedOutError(SynapseError): def cancelled_to_request_timed_out_error(value): """Turns CancelledErrors into RequestTimedOutErrors. - For use with deferred.addTimeout() + For use with async.add_timeout_to_deferred """ if isinstance(value, failure.Failure): value.trap(CancelledError) diff --git a/synapse/http/client.py b/synapse/http/client.py index 35c8d51e71..62309c3365 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -20,6 +20,7 @@ from synapse.api.errors import ( CodeMessageException, MatrixCodeMessageException, SynapseError, Codes, ) from synapse.http import cancelled_to_request_timed_out_error +from synapse.util.async import add_timeout_to_deferred from synapse.util.caches import CACHE_SIZE_FACTOR from synapse.util.logcontext import make_deferred_yieldable import synapse.metrics @@ -102,8 +103,9 @@ class SimpleHttpClient(object): request_deferred = self.agent.request( method, uri, *args, **kwargs ) - request_deferred.addTimeout( - 60, reactor, cancelled_to_request_timed_out_error, + add_timeout_to_deferred( + request_deferred, + 60, cancelled_to_request_timed_out_error, ) response = yield make_deferred_yieldable(request_deferred) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index fe4b1636a6..30036fe81c 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -22,7 +22,7 @@ from twisted.web._newclient import ResponseDone from synapse.http import cancelled_to_request_timed_out_error from synapse.http.endpoint import matrix_federation_endpoint import synapse.metrics -from synapse.util.async import sleep +from synapse.util.async import sleep, add_timeout_to_deferred from synapse.util import logcontext from synapse.util.logcontext import make_deferred_yieldable import synapse.util.retryutils @@ -193,9 +193,10 @@ class MatrixFederationHttpClient(object): Headers(headers_dict), producer ) - request_deferred.addTimeout( + add_timeout_to_deferred( + request_deferred, timeout / 1000. if timeout else 60, - reactor, cancelled_to_request_timed_out_error, + cancelled_to_request_timed_out_error, ) response = yield make_deferred_yieldable( request_deferred, diff --git a/synapse/notifier.py b/synapse/notifier.py index 1e4f78b993..a1c06e8fca 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -13,15 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer, reactor -from twisted.internet.defer import TimeoutError +from twisted.internet import defer from synapse.api.constants import EventTypes, Membership from synapse.api.errors import AuthError from synapse.handlers.presence import format_user_presence_state from synapse.util.logutils import log_function -from synapse.util.async import ObservableDeferred +from synapse.util.async import ( + ObservableDeferred, add_timeout_to_deferred, + DeferredTimeoutError, +) from synapse.util.logcontext import PreserveLoggingContext, preserve_fn from synapse.util.metrics import Measure from synapse.types import StreamToken @@ -332,8 +334,9 @@ class Notifier(object): # Now we wait for the _NotifierUserStream to be told there # is a new token. listener = user_stream.new_listener(prev_token) - listener.deferred.addTimeout( - (end_time - now) / 1000., reactor, + add_timeout_to_deferred( + listener.deferred, + (end_time - now) / 1000., ) with PreserveLoggingContext(): yield listener.deferred @@ -347,7 +350,7 @@ class Notifier(object): # Update the prev_token to the current_token since nothing # has happened between the old prev_token and the current_token prev_token = current_token - except TimeoutError: + except DeferredTimeoutError: break except defer.CancelledError: break @@ -552,11 +555,14 @@ class Notifier(object): if end_time <= now: break - listener.deferred.addTimeout((end_time - now) / 1000., reactor) + add_timeout_to_deferred( + listener.deferred.addTimeout, + (end_time - now) / 1000., + ) try: with PreserveLoggingContext(): yield listener.deferred - except TimeoutError: + except DeferredTimeoutError: break except defer.CancelledError: break diff --git a/synapse/util/async.py b/synapse/util/async.py index 0729bb2863..1df5c5600c 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -15,6 +15,8 @@ from twisted.internet import defer, reactor +from twisted.internet.defer import CancelledError +from twisted.python import failure from .logcontext import ( PreserveLoggingContext, make_deferred_yieldable, preserve_fn @@ -392,3 +394,68 @@ class ReadWriteLock(object): self.key_to_current_writer.pop(key) defer.returnValue(_ctx_manager()) + + +class DeferredTimeoutError(Exception): + """ + This error is raised by default when a L{Deferred} times out. + """ + + +def add_timeout_to_deferred(deferred, timeout, on_timeout_cancel=None): + """ + Add a timeout to a deferred by scheduling it to be cancelled after + timeout seconds. + + This is essentially a backport of deferred.addTimeout, which was introduced + in twisted 16.5. + + If the deferred gets timed out, it errbacks with a DeferredTimeoutError, + unless a cancelable function was passed to its initialization or unless + a different on_timeout_cancel callable is provided. + + Args: + deferred (defer.Deferred): deferred to be timed out + timeout (Number): seconds to time out after + + on_timeout_cancel (callable): A callable which is called immediately + after the deferred times out, and not if this deferred is + otherwise cancelled before the timeout. + + It takes an arbitrary value, which is the value of the deferred at + that exact point in time (probably a CancelledError Failure), and + the timeout. + + The default callable (if none is provided) will translate a + CancelledError Failure into a DeferredTimeoutError. + """ + timed_out = [False] + + def time_it_out(): + timed_out[0] = True + deferred.cancel() + + delayed_call = reactor.callLater(timeout, time_it_out) + + def convert_cancelled(value): + if timed_out[0]: + to_call = on_timeout_cancel or _cancelled_to_timed_out_error + return to_call(value, timeout) + return value + + deferred.addBoth(convert_cancelled) + + def cancel_timeout(result): + # stop the pending call to cancel the deferred if it's been fired + if delayed_call.active(): + delayed_call.cancel() + return result + + deferred.addBoth(cancel_timeout) + + +def _cancelled_to_timed_out_error(value, timeout): + if isinstance(value, failure.Failure): + value.trap(CancelledError) + raise DeferredTimeoutError(timeout, "Deferred") + return value From 2a13af23bc0561ab48e0a90528231c40ee209724 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Apr 2018 11:29:27 +0100 Subject: [PATCH 065/310] Use run_in_background in preference to preserve_fn While I was going through uses of preserve_fn for other PRs, I converted places which only use the wrapped function once to use run_in_background, to avoid creating the function object. --- synapse/app/federation_sender.py | 4 +-- synapse/app/pusher.py | 4 +-- synapse/app/synchrotron.py | 5 ++-- synapse/appservice/scheduler.py | 12 ++++---- synapse/crypto/keyring.py | 28 +++++++++++-------- synapse/federation/federation_client.py | 5 ++-- synapse/groups/attestations.py | 4 +-- synapse/handlers/appservice.py | 5 +++- synapse/handlers/e2e_keys.py | 6 ++-- synapse/handlers/federation.py | 16 +++++++---- synapse/handlers/initial_sync.py | 12 ++++---- synapse/handlers/message.py | 5 ++-- synapse/handlers/typing.py | 7 +++-- synapse/push/pusherpool.py | 20 +++++++------ synapse/rest/media/v1/preview_url_resource.py | 5 ++-- synapse/storage/events_worker.py | 5 ++-- synapse/storage/stream.py | 5 ++-- synapse/util/async.py | 4 +-- synapse/util/file_consumer.py | 6 ++-- synapse/util/logcontext.py | 2 +- synapse/util/ratelimitutils.py | 4 +-- synapse/util/retryutils.py | 4 +-- 22 files changed, 97 insertions(+), 71 deletions(-) diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py index 0cc3331519..c6daa0d43f 100644 --- a/synapse/app/federation_sender.py +++ b/synapse/app/federation_sender.py @@ -38,7 +38,7 @@ from synapse.server import HomeServer from synapse.storage.engines import create_engine from synapse.util.async import Linearizer from synapse.util.httpresourcetree import create_resource_tree -from synapse.util.logcontext import LoggingContext, preserve_fn +from synapse.util.logcontext import LoggingContext, run_in_background from synapse.util.manhole import manhole from synapse.util.versionstring import get_version_string from twisted.internet import defer, reactor @@ -229,7 +229,7 @@ class FederationSenderHandler(object): # presence, typing, etc. if stream_name == "federation": send_queue.process_rows_for_federation(self.federation_sender, rows) - preserve_fn(self.update_token)(token) + run_in_background(self.update_token, token) # We also need to poke the federation sender when new events happen elif stream_name == "events": diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index d5c3a85195..8bd5c0c2b7 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -33,7 +33,7 @@ from synapse.server import HomeServer from synapse.storage import DataStore from synapse.storage.engines import create_engine from synapse.util.httpresourcetree import create_resource_tree -from synapse.util.logcontext import LoggingContext, preserve_fn +from synapse.util.logcontext import LoggingContext, run_in_background from synapse.util.manhole import manhole from synapse.util.versionstring import get_version_string from twisted.internet import defer, reactor @@ -140,7 +140,7 @@ class PusherReplicationHandler(ReplicationClientHandler): def on_rdata(self, stream_name, token, rows): super(PusherReplicationHandler, self).on_rdata(stream_name, token, rows) - preserve_fn(self.poke_pushers)(stream_name, token, rows) + run_in_background(self.poke_pushers, stream_name, token, rows) @defer.inlineCallbacks def poke_pushers(self, stream_name, token, rows): diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index 2fddcd935a..0c4ccc58bc 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -51,7 +51,7 @@ from synapse.storage.engines import create_engine from synapse.storage.presence import UserPresenceState from synapse.storage.roommember import RoomMemberStore from synapse.util.httpresourcetree import create_resource_tree -from synapse.util.logcontext import LoggingContext, preserve_fn +from synapse.util.logcontext import LoggingContext, run_in_background from synapse.util.manhole import manhole from synapse.util.stringutils import random_string from synapse.util.versionstring import get_version_string @@ -327,8 +327,7 @@ class SyncReplicationHandler(ReplicationClientHandler): def on_rdata(self, stream_name, token, rows): super(SyncReplicationHandler, self).on_rdata(stream_name, token, rows) - - preserve_fn(self.process_and_notify)(stream_name, token, rows) + run_in_background(self.process_and_notify, stream_name, token, rows) def get_streams_to_replicate(self): args = super(SyncReplicationHandler, self).get_streams_to_replicate() diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 6da315473d..ba1631b5c8 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -51,7 +51,7 @@ components. from twisted.internet import defer from synapse.appservice import ApplicationServiceState -from synapse.util.logcontext import preserve_fn +from synapse.util.logcontext import run_in_background from synapse.util.metrics import Measure import logging @@ -106,7 +106,7 @@ class _ServiceQueuer(object): def enqueue(self, service, event): # if this service isn't being sent something self.queued_events.setdefault(service.id, []).append(event) - preserve_fn(self._send_request)(service) + run_in_background(self._send_request, service) @defer.inlineCallbacks def _send_request(self, service): @@ -152,10 +152,10 @@ class _TransactionController(object): if sent: yield txn.complete(self.store) else: - preserve_fn(self._start_recoverer)(service) - except Exception as e: - logger.exception(e) - preserve_fn(self._start_recoverer)(service) + run_in_background(self._start_recoverer, service) + except Exception: + logger.exception("Error creating appservice transaction") + run_in_background(self._start_recoverer, service) @defer.inlineCallbacks def on_recovered(self, recoverer): diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index fce83d445f..38944a7326 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -19,7 +19,8 @@ from synapse.api.errors import SynapseError, Codes from synapse.util import unwrapFirstError, logcontext from synapse.util.logcontext import ( PreserveLoggingContext, - preserve_fn + preserve_fn, + run_in_background, ) from synapse.util.metrics import Measure @@ -127,7 +128,7 @@ class Keyring(object): verify_requests.append(verify_request) - preserve_fn(self._start_key_lookups)(verify_requests) + run_in_background(self._start_key_lookups, verify_requests) # Pass those keys to handle_key_deferred so that the json object # signatures can be verified @@ -313,7 +314,7 @@ class Keyring(object): if not verify_request.deferred.called: verify_request.deferred.errback(err) - preserve_fn(do_iterations)().addErrback(on_err) + run_in_background(do_iterations).addErrback(on_err) @defer.inlineCallbacks def get_keys_from_store(self, server_name_and_key_ids): @@ -329,8 +330,9 @@ class Keyring(object): """ res = yield logcontext.make_deferred_yieldable(defer.gatherResults( [ - preserve_fn(self.store.get_server_verify_keys)( - server_name, key_ids + run_in_background( + self.store.get_server_verify_keys, + server_name, key_ids, ).addCallback(lambda ks, server: (server, ks), server_name) for server_name, key_ids in server_name_and_key_ids ], @@ -358,7 +360,7 @@ class Keyring(object): results = yield logcontext.make_deferred_yieldable(defer.gatherResults( [ - preserve_fn(get_key)(p_name, p_keys) + run_in_background(get_key, p_name, p_keys) for p_name, p_keys in self.perspective_servers.items() ], consumeErrors=True, @@ -398,7 +400,7 @@ class Keyring(object): results = yield logcontext.make_deferred_yieldable(defer.gatherResults( [ - preserve_fn(get_key)(server_name, key_ids) + run_in_background(get_key, server_name, key_ids) for server_name, key_ids in server_name_and_key_ids ], consumeErrors=True, @@ -481,7 +483,8 @@ class Keyring(object): yield logcontext.make_deferred_yieldable(defer.gatherResults( [ - preserve_fn(self.store_keys)( + run_in_background( + self.store_keys, server_name=server_name, from_server=perspective_name, verify_keys=response_keys, @@ -539,7 +542,8 @@ class Keyring(object): yield logcontext.make_deferred_yieldable(defer.gatherResults( [ - preserve_fn(self.store_keys)( + run_in_background( + self.store_keys, server_name=key_server_name, from_server=server_name, verify_keys=verify_keys, @@ -615,7 +619,8 @@ class Keyring(object): yield logcontext.make_deferred_yieldable(defer.gatherResults( [ - preserve_fn(self.store.store_server_keys_json)( + run_in_background( + self.store.store_server_keys_json, server_name=server_name, key_id=key_id, from_server=server_name, @@ -716,7 +721,8 @@ class Keyring(object): # TODO(markjh): Store whether the keys have expired. return logcontext.make_deferred_yieldable(defer.gatherResults( [ - preserve_fn(self.store.store_server_verify_key)( + run_in_background( + self.store.store_server_verify_key, server_name, server_name, key.time_added, key ) for key_id, key in verify_keys.items() diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 8e2c0c4cd2..8adc60863e 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -33,7 +33,7 @@ from synapse.federation.federation_base import ( import synapse.metrics from synapse.util import logcontext, unwrapFirstError from synapse.util.caches.expiringcache import ExpiringCache -from synapse.util.logcontext import make_deferred_yieldable, preserve_fn +from synapse.util.logcontext import make_deferred_yieldable, run_in_background from synapse.util.logutils import log_function from synapse.util.retryutils import NotRetryingDestination @@ -417,7 +417,8 @@ class FederationClient(FederationBase): batch = set(missing_events[i:i + batch_size]) deferreds = [ - preserve_fn(self.get_pdu)( + run_in_background( + self.get_pdu, destinations=random_server_list(), event_id=e_id, ) diff --git a/synapse/groups/attestations.py b/synapse/groups/attestations.py index 1fb709e6c3..5f53f17954 100644 --- a/synapse/groups/attestations.py +++ b/synapse/groups/attestations.py @@ -42,7 +42,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError from synapse.types import get_domain_from_id -from synapse.util.logcontext import preserve_fn +from synapse.util.logcontext import run_in_background from signedjson.sign import sign_json @@ -192,4 +192,4 @@ class GroupAttestionRenewer(object): group_id = row["group_id"] user_id = row["user_id"] - preserve_fn(_renew_attestation)(group_id, user_id) + run_in_background(_renew_attestation, group_id, user_id) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 0245197c02..6cc2388306 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -198,7 +198,10 @@ class ApplicationServicesHandler(object): services = yield self._get_services_for_3pn(protocol) results = yield make_deferred_yieldable(defer.DeferredList([ - preserve_fn(self.appservice_api.query_3pe)(service, kind, protocol, fields) + run_in_background( + self.appservice_api.query_3pe, + service, kind, protocol, fields, + ) for service in services ], consumeErrors=True)) diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index 325c0c4a9f..fc958404a1 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -24,7 +24,7 @@ from synapse.api.errors import ( SynapseError, CodeMessageException, FederationDeniedError, ) from synapse.types import get_domain_from_id, UserID -from synapse.util.logcontext import preserve_fn, make_deferred_yieldable +from synapse.util.logcontext import make_deferred_yieldable, run_in_background from synapse.util.retryutils import NotRetryingDestination logger = logging.getLogger(__name__) @@ -139,7 +139,7 @@ class E2eKeysHandler(object): failures[destination] = _exception_to_failure(e) yield make_deferred_yieldable(defer.gatherResults([ - preserve_fn(do_remote_query)(destination) + run_in_background(do_remote_query, destination) for destination in remote_queries_not_in_cache ])) @@ -242,7 +242,7 @@ class E2eKeysHandler(object): failures[destination] = _exception_to_failure(e) yield make_deferred_yieldable(defer.gatherResults([ - preserve_fn(claim_client_keys)(destination) + run_in_background(claim_client_keys, destination) for destination in remote_queries ])) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index ae7e0d6da2..c66ca0f381 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -637,7 +637,8 @@ class FederationHandler(BaseHandler): results = yield logcontext.make_deferred_yieldable(defer.gatherResults( [ - logcontext.preserve_fn(self.replication_layer.get_pdu)( + logcontext.run_in_background( + self.replication_layer.get_pdu, [dest], event_id, outlier=True, @@ -1023,7 +1024,7 @@ class FederationHandler(BaseHandler): # lots of requests for missing prev_events which we do actually # have. Hence we fire off the deferred, but don't wait for it. - logcontext.preserve_fn(self._handle_queued_pdus)(room_queue) + logcontext.run_in_background(self._handle_queued_pdus, room_queue) defer.returnValue(True) @@ -1523,8 +1524,9 @@ class FederationHandler(BaseHandler): if not backfilled: # this intentionally does not yield: we don't care about the result # and don't need to wait for it. - logcontext.preserve_fn(self.pusher_pool.on_new_notifications)( - event_stream_id, max_stream_id + logcontext.run_in_background( + self.pusher_pool.on_new_notifications, + event_stream_id, max_stream_id, ) defer.returnValue((context, event_stream_id, max_stream_id)) @@ -1538,7 +1540,8 @@ class FederationHandler(BaseHandler): """ contexts = yield logcontext.make_deferred_yieldable(defer.gatherResults( [ - logcontext.preserve_fn(self._prep_event)( + logcontext.run_in_background( + self._prep_event, origin, ev_info["event"], state=ev_info.get("state"), @@ -1867,7 +1870,8 @@ class FederationHandler(BaseHandler): different_events = yield logcontext.make_deferred_yieldable( defer.gatherResults([ - logcontext.preserve_fn(self.store.get_event)( + logcontext.run_in_background( + self.store.get_event, d, allow_none=True, allow_rejected=False, diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py index c5267b4b84..cd33a86599 100644 --- a/synapse/handlers/initial_sync.py +++ b/synapse/handlers/initial_sync.py @@ -27,7 +27,7 @@ from synapse.types import ( from synapse.util import unwrapFirstError from synapse.util.async import concurrently_execute from synapse.util.caches.snapshot_cache import SnapshotCache -from synapse.util.logcontext import make_deferred_yieldable, preserve_fn +from synapse.util.logcontext import make_deferred_yieldable, run_in_background from synapse.visibility import filter_events_for_client from ._base import BaseHandler @@ -166,7 +166,8 @@ class InitialSyncHandler(BaseHandler): (messages, token), current_state = yield make_deferred_yieldable( defer.gatherResults( [ - preserve_fn(self.store.get_recent_events_for_room)( + run_in_background( + self.store.get_recent_events_for_room, event.room_id, limit=limit, end_token=room_end_token, @@ -391,9 +392,10 @@ class InitialSyncHandler(BaseHandler): presence, receipts, (messages, token) = yield defer.gatherResults( [ - preserve_fn(get_presence)(), - preserve_fn(get_receipts)(), - preserve_fn(self.store.get_recent_events_for_room)( + run_in_background(get_presence), + run_in_background(get_receipts), + run_in_background( + self.store.get_recent_events_for_room, room_id, limit=limit, end_token=now_token.room_key, diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 21628a8540..244b98dd8d 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -850,7 +850,8 @@ class EventCreationHandler(object): # this intentionally does not yield: we don't care about the result # and don't need to wait for it. - preserve_fn(self.pusher_pool.on_new_notifications)( + run_in_background( + self.pusher_pool.on_new_notifications, event_stream_id, max_stream_id ) @@ -862,7 +863,7 @@ class EventCreationHandler(object): extra_users=extra_users ) - preserve_fn(_notify)() + run_in_background(_notify) if event.type == EventTypes.Message: presence = self.hs.get_presence_handler() diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 77c0cf146f..19cde70adf 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -16,7 +16,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, AuthError -from synapse.util.logcontext import preserve_fn +from synapse.util.logcontext import run_in_background from synapse.util.metrics import Measure from synapse.util.wheel_timer import WheelTimer from synapse.types import UserID, get_domain_from_id @@ -97,7 +97,8 @@ class TypingHandler(object): if self.hs.is_mine_id(member.user_id): last_fed_poke = self._member_last_federation_poke.get(member, None) if not last_fed_poke or last_fed_poke + FEDERATION_PING_INTERVAL <= now: - preserve_fn(self._push_remote)( + run_in_background( + self._push_remote, member=member, typing=True ) @@ -196,7 +197,7 @@ class TypingHandler(object): def _push_update(self, member, typing): if self.hs.is_mine_id(member.user_id): # Only send updates for changes to our own users. - preserve_fn(self._push_remote)(member, typing) + run_in_background(self._push_remote, member, typing) self._push_update_local( member=member, diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 134e89b371..7bb5733090 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -14,13 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging + from twisted.internet import defer -from .pusher import PusherFactory -from synapse.util.logcontext import make_deferred_yieldable, preserve_fn +from synapse.push.pusher import PusherFactory from synapse.util.async import run_on_reactor - -import logging +from synapse.util.logcontext import make_deferred_yieldable, run_in_background logger = logging.getLogger(__name__) @@ -137,8 +137,9 @@ class PusherPool: if u in self.pushers: for p in self.pushers[u].values(): deferreds.append( - preserve_fn(p.on_new_notifications)( - min_stream_id, max_stream_id + run_in_background( + p.on_new_notifications, + min_stream_id, max_stream_id, ) ) @@ -164,7 +165,10 @@ class PusherPool: if u in self.pushers: for p in self.pushers[u].values(): deferreds.append( - preserve_fn(p.on_new_receipts)(min_stream_id, max_stream_id) + run_in_background( + p.on_new_receipts, + min_stream_id, max_stream_id, + ) ) yield make_deferred_yieldable(defer.gatherResults(deferreds)) @@ -207,7 +211,7 @@ class PusherPool: if appid_pushkey in byuser: byuser[appid_pushkey].on_stop() byuser[appid_pushkey] = p - preserve_fn(p.on_started)() + run_in_background(p.on_started) logger.info("Started pushers") diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py index 0fc21540c6..9290d7946f 100644 --- a/synapse/rest/media/v1/preview_url_resource.py +++ b/synapse/rest/media/v1/preview_url_resource.py @@ -35,7 +35,7 @@ from ._base import FileInfo from synapse.api.errors import ( SynapseError, Codes, ) -from synapse.util.logcontext import preserve_fn, make_deferred_yieldable +from synapse.util.logcontext import make_deferred_yieldable, run_in_background from synapse.util.stringutils import random_string from synapse.util.caches.expiringcache import ExpiringCache from synapse.http.client import SpiderHttpClient @@ -144,7 +144,8 @@ class PreviewUrlResource(Resource): observable = self._cache.get(url) if not observable: - download = preserve_fn(self._do_preview)( + download = run_in_background( + self._do_preview, url, requester.user, ts, ) observable = ObservableDeferred( diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py index a937b9bceb..ba834854e1 100644 --- a/synapse/storage/events_worker.py +++ b/synapse/storage/events_worker.py @@ -20,7 +20,7 @@ from synapse.events import FrozenEvent from synapse.events.utils import prune_event from synapse.util.logcontext import ( - preserve_fn, PreserveLoggingContext, make_deferred_yieldable + PreserveLoggingContext, make_deferred_yieldable, run_in_background, ) from synapse.util.metrics import Measure from synapse.api.errors import SynapseError @@ -319,7 +319,8 @@ class EventsWorkerStore(SQLBaseStore): res = yield make_deferred_yieldable(defer.gatherResults( [ - preserve_fn(self._get_event_from_row)( + run_in_background( + self._get_event_from_row, row["internal_metadata"], row["json"], row["redacts"], rejected_reason=row["rejects"], ) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 2956c3b3e0..5b245a936c 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -41,7 +41,7 @@ from synapse.storage.events import EventsWorkerStore from synapse.util.caches.descriptors import cached from synapse.types import RoomStreamToken from synapse.util.caches.stream_change_cache import StreamChangeCache -from synapse.util.logcontext import make_deferred_yieldable, preserve_fn +from synapse.util.logcontext import make_deferred_yieldable, run_in_background from synapse.storage.engines import PostgresEngine, Sqlite3Engine import abc @@ -198,7 +198,8 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): room_ids = list(room_ids) for rm_ids in (room_ids[i:i + 20] for i in xrange(0, len(room_ids), 20)): res = yield make_deferred_yieldable(defer.gatherResults([ - preserve_fn(self.get_room_events_stream_for_room)( + run_in_background( + self.get_room_events_stream_for_room, room_id, from_key, to_key, limit, order=order, ) for room_id in rm_ids diff --git a/synapse/util/async.py b/synapse/util/async.py index 0729bb2863..bd07067328 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -17,7 +17,7 @@ from twisted.internet import defer, reactor from .logcontext import ( - PreserveLoggingContext, make_deferred_yieldable, preserve_fn + PreserveLoggingContext, make_deferred_yieldable, run_in_background ) from synapse.util import logcontext, unwrapFirstError @@ -161,7 +161,7 @@ def concurrently_execute(func, args, limit): pass return logcontext.make_deferred_yieldable(defer.gatherResults([ - preserve_fn(_concurrently_execute_inner)() + run_in_background(_concurrently_execute_inner) for _ in xrange(limit) ], consumeErrors=True)).addErrback(unwrapFirstError) diff --git a/synapse/util/file_consumer.py b/synapse/util/file_consumer.py index 3c8a165331..3380970e4e 100644 --- a/synapse/util/file_consumer.py +++ b/synapse/util/file_consumer.py @@ -15,7 +15,7 @@ from twisted.internet import threads, reactor -from synapse.util.logcontext import make_deferred_yieldable, preserve_fn +from synapse.util.logcontext import make_deferred_yieldable, run_in_background from six.moves import queue @@ -70,7 +70,9 @@ class BackgroundFileConsumer(object): self._producer = producer self.streaming = streaming - self._finished_deferred = preserve_fn(threads.deferToThread)(self._writer) + self._finished_deferred = run_in_background( + threads.deferToThread, self._writer + ) if not streaming: self._producer.resumeProducing() diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index d59adc236e..c2edf87e58 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -341,7 +341,7 @@ def make_deferred_yieldable(deferred): returning a deferred. Then, when the deferred completes, restores the current logcontext before running callbacks/errbacks. - (This is more-or-less the opposite operation to preserve_fn.) + (This is more-or-less the opposite operation to run_in_background.) """ if isinstance(deferred, defer.Deferred) and not deferred.called: prev_context = LoggingContext.set_current_context(LoggingContext.sentinel) diff --git a/synapse/util/ratelimitutils.py b/synapse/util/ratelimitutils.py index 1101881a2d..18424f6c36 100644 --- a/synapse/util/ratelimitutils.py +++ b/synapse/util/ratelimitutils.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.errors import LimitExceededError from synapse.util.async import sleep -from synapse.util.logcontext import preserve_fn +from synapse.util.logcontext import run_in_background import collections import contextlib @@ -150,7 +150,7 @@ class _PerHostRatelimiter(object): "Ratelimit [%s]: sleeping req", id(request_id), ) - ret_defer = preserve_fn(sleep)(self.sleep_msec / 1000.0) + ret_defer = run_in_background(sleep, self.sleep_msec / 1000.0) self.sleeping_requests.add(request_id) diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index 47b0bb5eb3..4e93f69d3a 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -203,8 +203,8 @@ class RetryDestinationLimiter(object): ) except Exception: logger.exception( - "Failed to store set_destination_retry_timings", + "Failed to store destination_retry_timings", ) # we deliberately do this in the background. - synapse.util.logcontext.preserve_fn(store_retry_timings)() + synapse.util.logcontext.run_in_background(store_retry_timings) From 453adf00b67fc9156050678640caa38a71181ebf Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Apr 2018 14:32:08 +0100 Subject: [PATCH 066/310] pep8; remove spurious import --- synapse/handlers/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 0f26b91862..23502eda70 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -31,7 +31,7 @@ from synapse.types import ( UserID, RoomAlias, RoomStreamToken, ) from synapse.util.async import run_on_reactor, ReadWriteLock, Limiter -from synapse.util.logcontext import preserve_fn, run_in_background +from synapse.util.logcontext import run_in_background from synapse.util.metrics import measure_func from synapse.util.frozenutils import frozendict_json_encoder from synapse.util.stringutils import random_string From 4f5694e2cecc8530d09fd00e8bb6f165764a6042 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Fri, 27 Apr 2018 16:22:26 +0200 Subject: [PATCH 067/310] Add py3 tests to tox with folders that work It's just a few tests, but it will at least prevent a few files from regressing. Also, it makes it easiert to check your code against py36 while writing it. Signed-off-by: Adrian Tschira --- tox.ini | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index f408defc8f..c6cee92568 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = packaging, py27, pep8 +envlist = packaging, py27, py36 pep8 [testenv] deps = @@ -46,6 +46,14 @@ commands = # ) usedevelop=true +[testenv:py36] +usedevelop=true +commands = + /usr/bin/find "{toxinidir}" -name '*.pyc' -delete + coverage run {env:COVERAGE_OPTS:} --source="{toxinidir}/synapse" \ + "{envbindir}/trial" {env:TRIAL_FLAGS:} {posargs:tests/metrics} {env:TOXSUFFIX:} + {env:DUMP_COVERAGE_COMMAND:coverage report -m} + [testenv:packaging] deps = check-manifest From a376d8f7615e9715d6c5c3fa2d95f9d48ff87a61 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Sat, 28 Apr 2018 12:29:02 +0200 Subject: [PATCH 068/310] open log_config in text mode too Signed-off-by: Adrian Tschira --- synapse/config/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 3f70039acd..6a7228dc2f 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -117,7 +117,7 @@ class LoggingConfig(Config): log_config = config.get("log_config") if log_config and not os.path.exists(log_config): log_file = self.abspath("homeserver.log") - with open(log_config, "wb") as log_config_file: + with open(log_config, "w") as log_config_file: log_config_file.write( DEFAULT_LOG_CONFIG.substitute(log_file=log_file) ) From cdb4647a80dfccf4f8865a04c28bfaffa378ef10 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Sat, 28 Apr 2018 13:04:40 +0200 Subject: [PATCH 069/310] Don't yield in list comprehensions I've tried to grep for more of this with no success. Signed-off-by: Adrian Tschira --- synapse/handlers/appservice.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 0245197c02..b7776693de 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -259,11 +259,15 @@ class ApplicationServicesHandler(object): event based on the service regex. """ services = self.store.get_app_services() - interested_list = [ - s for s in services if ( - yield s.is_interested(event, self.store) - ) - ] + + # we can't use a list comprehension here. Since python 3, list + # comprehensions use a generator internally. This means you can't yield + # inside of a list comprehension anymore. + interested_list = [] + for s in services: + if (yield s.is_interested(event, self.store)): + interested_list.append(s) + defer.returnValue(interested_list) def _get_services_for_user(self, user_id): From 57b58e2174f120fb13fbe2f6d57e8647b69921ec Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Sat, 28 Apr 2018 12:17:56 +0200 Subject: [PATCH 070/310] make imports local Signed-off-by: Adrian Tschira --- synapse/push/httppusher.py | 4 ++-- synapse/push/pusher.py | 2 +- synapse/replication/tcp/protocol.py | 4 ++-- synapse/replication/tcp/resource.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 1420d378ef..b077e1a446 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -18,8 +18,8 @@ import logging from twisted.internet import defer, reactor from twisted.internet.error import AlreadyCalled, AlreadyCancelled -import push_rule_evaluator -import push_tools +from . import push_rule_evaluator +from . import push_tools import synapse from synapse.push import PusherConfigException from synapse.util.logcontext import LoggingContext diff --git a/synapse/push/pusher.py b/synapse/push/pusher.py index 71576330a9..5aa6667e91 100644 --- a/synapse/push/pusher.py +++ b/synapse/push/pusher.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from httppusher import HttpPusher +from .httppusher import HttpPusher import logging logger = logging.getLogger(__name__) diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py index 0a9a290af4..d7d38464b2 100644 --- a/synapse/replication/tcp/protocol.py +++ b/synapse/replication/tcp/protocol.py @@ -53,12 +53,12 @@ from twisted.internet import defer from twisted.protocols.basic import LineOnlyReceiver from twisted.python.failure import Failure -from commands import ( +from .commands import ( COMMAND_MAP, VALID_CLIENT_COMMANDS, VALID_SERVER_COMMANDS, ErrorCommand, ServerCommand, RdataCommand, PositionCommand, PingCommand, NameCommand, ReplicateCommand, UserSyncCommand, SyncCommand, ) -from streams import STREAMS_MAP +from .streams import STREAMS_MAP from synapse.util.stringutils import random_string from synapse.metrics.metric import CounterMetric diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index 786c3fe864..a41af4fd6c 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -18,8 +18,8 @@ from twisted.internet import defer, reactor from twisted.internet.protocol import Factory -from streams import STREAMS_MAP, FederationStream -from protocol import ServerReplicationStreamProtocol +from .streams import STREAMS_MAP, FederationStream +from .protocol import ServerReplicationStreamProtocol from synapse.util.metrics import Measure, measure_func From 94f4d7f49ed322dac38e8946bba590ac41c0f40f Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Sat, 28 Apr 2018 11:46:48 +0200 Subject: [PATCH 071/310] move httplib import to six --- synapse/handlers/federation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 260df025f9..24ff6782c5 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -16,7 +16,6 @@ """Contains handlers for federation events.""" -import httplib import itertools import logging import sys @@ -24,6 +23,7 @@ import sys from signedjson.key import decode_verify_key_bytes from signedjson.sign import verify_signed_json import six +from six.moves import http_client from twisted.internet import defer from unpaddedbase64 import decode_base64 @@ -889,7 +889,7 @@ class FederationHandler(BaseHandler): logger.warn("Rejecting event %s which has %i prev_events", ev.event_id, len(ev.prev_events)) raise SynapseError( - httplib.BAD_REQUEST, + http_client.BAD_REQUEST, "Too many prev_events", ) @@ -897,7 +897,7 @@ class FederationHandler(BaseHandler): logger.warn("Rejecting event %s which has %i auth_events", ev.event_id, len(ev.auth_events)) raise SynapseError( - httplib.BAD_REQUEST, + http_client.BAD_REQUEST, "Too many auth_events", ) From 4f2f5171b752169e265b5d93c9b4e788171b5326 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Sat, 28 Apr 2018 11:39:37 +0200 Subject: [PATCH 072/310] replace stringIO imports --- synapse/http/client.py | 4 ++-- synapse/util/logformatter.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/http/client.py b/synapse/http/client.py index 62309c3365..70a19d9b74 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -40,7 +40,7 @@ from twisted.web.http import PotentialDataLoss from twisted.web.http_headers import Headers from twisted.web._newclient import ResponseDone -from StringIO import StringIO +from six import StringIO import simplejson as json import logging @@ -507,7 +507,7 @@ class SpiderHttpClient(SimpleHttpClient): reactor, SpiderEndpointFactory(hs) ) - ), [('gzip', GzipDecoder)] + ), [(b'gzip', GzipDecoder)] ) # We could look like Chrome: # self.user_agent = ("Mozilla/5.0 (%s) (KHTML, like Gecko) diff --git a/synapse/util/logformatter.py b/synapse/util/logformatter.py index cdbc4bffd7..59ab3c6968 100644 --- a/synapse/util/logformatter.py +++ b/synapse/util/logformatter.py @@ -14,7 +14,7 @@ # limitations under the License. -import StringIO +from six import StringIO import logging import traceback From d82b6ea9e68150aece1fc46cb0821b31cf728910 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Sat, 28 Apr 2018 13:57:00 +0200 Subject: [PATCH 073/310] Move more xrange to six plus a bonus next() Signed-off-by: Adrian Tschira --- synapse/federation/federation_client.py | 4 +++- synapse/handlers/room_list.py | 4 +++- synapse/storage/registration.py | 4 +++- synapse/storage/schema/delta/30/as_users.py | 4 +++- synapse/storage/stream.py | 4 +++- synapse/storage/tags.py | 4 +++- synapse/util/async.py | 6 ++++-- synapse/util/stringutils.py | 5 +++-- synapse/util/wheel_timer.py | 4 +++- 9 files changed, 28 insertions(+), 11 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 8e2c0c4cd2..a2067fc390 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -19,6 +19,8 @@ import itertools import logging import random +from six.moves import range + from twisted.internet import defer from synapse.api.constants import Membership @@ -413,7 +415,7 @@ class FederationClient(FederationBase): batch_size = 20 missing_events = list(missing_events) - for i in xrange(0, len(missing_events), batch_size): + for i in range(0, len(missing_events), batch_size): batch = set(missing_events[i:i + batch_size]) deferreds = [ diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index add3f9b009..5757bb7f8a 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -15,6 +15,8 @@ from twisted.internet import defer +from six.moves import range + from ._base import BaseHandler from synapse.api.constants import ( @@ -200,7 +202,7 @@ class RoomListHandler(BaseHandler): step = len(rooms_to_scan) if len(rooms_to_scan) != 0 else 1 chunk = [] - for i in xrange(0, len(rooms_to_scan), step): + for i in range(0, len(rooms_to_scan), step): batch = rooms_to_scan[i:i + step] logger.info("Processing %i rooms for result", len(batch)) yield concurrently_execute( diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 6b557ca0cf..a50717db2d 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -22,6 +22,8 @@ from synapse.storage import background_updates from synapse.storage._base import SQLBaseStore from synapse.util.caches.descriptors import cached, cachedInlineCallbacks +from six.moves import range + class RegistrationWorkerStore(SQLBaseStore): @cached() @@ -469,7 +471,7 @@ class RegistrationStore(RegistrationWorkerStore, match = regex.search(user_id) if match: found.add(int(match.group(1))) - for i in xrange(len(found) + 1): + for i in range(len(found) + 1): if i not in found: return i diff --git a/synapse/storage/schema/delta/30/as_users.py b/synapse/storage/schema/delta/30/as_users.py index c53e53c94f..85bd1a2006 100644 --- a/synapse/storage/schema/delta/30/as_users.py +++ b/synapse/storage/schema/delta/30/as_users.py @@ -14,6 +14,8 @@ import logging from synapse.config.appservice import load_appservices +from six.moves import range + logger = logging.getLogger(__name__) @@ -58,7 +60,7 @@ def run_upgrade(cur, database_engine, config, *args, **kwargs): for as_id, user_ids in owned.items(): n = 100 - user_chunks = (user_ids[i:i + 100] for i in xrange(0, len(user_ids), n)) + user_chunks = (user_ids[i:i + 100] for i in range(0, len(user_ids), n)) for chunk in user_chunks: cur.execute( database_engine.convert_param_style( diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 3b8b539993..52c90d8e04 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -47,6 +47,8 @@ from synapse.storage.engines import PostgresEngine, Sqlite3Engine import abc import logging +from six.moves import range + logger = logging.getLogger(__name__) @@ -196,7 +198,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): results = {} room_ids = list(room_ids) - for rm_ids in (room_ids[i:i + 20] for i in xrange(0, len(room_ids), 20)): + for rm_ids in (room_ids[i:i + 20] for i in range(0, len(room_ids), 20)): res = yield make_deferred_yieldable(defer.gatherResults([ preserve_fn(self.get_room_events_stream_for_room)( room_id, from_key, to_key, limit, order=order, diff --git a/synapse/storage/tags.py b/synapse/storage/tags.py index 13bff9f055..6671d3cfca 100644 --- a/synapse/storage/tags.py +++ b/synapse/storage/tags.py @@ -22,6 +22,8 @@ from twisted.internet import defer import simplejson as json import logging +from six.moves import range + logger = logging.getLogger(__name__) @@ -98,7 +100,7 @@ class TagsWorkerStore(AccountDataWorkerStore): batch_size = 50 results = [] - for i in xrange(0, len(tag_ids), batch_size): + for i in range(0, len(tag_ids), batch_size): tags = yield self.runInteraction( "get_all_updated_tag_content", get_tag_content, diff --git a/synapse/util/async.py b/synapse/util/async.py index 1df5c5600c..cb53c31123 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -27,6 +27,8 @@ from contextlib import contextmanager import logging +from six.moves import range + logger = logging.getLogger(__name__) @@ -158,13 +160,13 @@ def concurrently_execute(func, args, limit): def _concurrently_execute_inner(): try: while True: - yield func(it.next()) + yield func(next(it)) except StopIteration: pass return logcontext.make_deferred_yieldable(defer.gatherResults([ preserve_fn(_concurrently_execute_inner)() - for _ in xrange(limit) + for _ in range(limit) ], consumeErrors=True)).addErrback(unwrapFirstError) diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py index 95a6168e16..b98b9dc6e4 100644 --- a/synapse/util/stringutils.py +++ b/synapse/util/stringutils.py @@ -15,6 +15,7 @@ import random import string +from six.moves import range _string_with_symbols = ( string.digits + string.ascii_letters + ".,;:^&*-_+=#~@" @@ -22,12 +23,12 @@ _string_with_symbols = ( def random_string(length): - return ''.join(random.choice(string.ascii_letters) for _ in xrange(length)) + return ''.join(random.choice(string.ascii_letters) for _ in range(length)) def random_string_with_symbols(length): return ''.join( - random.choice(_string_with_symbols) for _ in xrange(length) + random.choice(_string_with_symbols) for _ in range(length) ) diff --git a/synapse/util/wheel_timer.py b/synapse/util/wheel_timer.py index b70f9a6b0a..7a9e45aca9 100644 --- a/synapse/util/wheel_timer.py +++ b/synapse/util/wheel_timer.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from six.moves import range + class _Entry(object): __slots__ = ["end_key", "queue"] @@ -68,7 +70,7 @@ class WheelTimer(object): # Add empty entries between the end of the current list and when we want # to insert. This ensures there are no gaps. self.entries.extend( - _Entry(key) for key in xrange(last_key, then_key + 1) + _Entry(key) for key in range(last_key, then_key + 1) ) self.entries[-1].queue.append(obj) From 42c89c8215533e3ef48cf8c9bf33595d4826b905 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 28 Apr 2018 22:27:30 +0100 Subject: [PATCH 074/310] make it work with sqlite --- .../schema/delta/48/group_unique_indexes.py | 54 +++++++++++++++++++ .../schema/delta/48/group_unique_indexes.sql | 34 ------------ 2 files changed, 54 insertions(+), 34 deletions(-) create mode 100644 synapse/storage/schema/delta/48/group_unique_indexes.py delete mode 100644 synapse/storage/schema/delta/48/group_unique_indexes.sql diff --git a/synapse/storage/schema/delta/48/group_unique_indexes.py b/synapse/storage/schema/delta/48/group_unique_indexes.py new file mode 100644 index 0000000000..c1f5881b3e --- /dev/null +++ b/synapse/storage/schema/delta/48/group_unique_indexes.py @@ -0,0 +1,54 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.storage.engines import PostgresEngine + +FIX_INDEXES = """ +-- rebuild indexes as uniques +DROP INDEX groups_invites_g_idx; +CREATE UNIQUE INDEX group_invites_g_idx ON group_invites(group_id, user_id); +DROP INDEX groups_users_g_idx; +CREATE UNIQUE INDEX group_users_g_idx ON group_users(group_id, user_id); + +-- rename other indexes to actually match their table names.. +DROP INDEX groups_users_u_idx; +CREATE INDEX group_users_u_idx ON group_users(user_id); +DROP INDEX groups_invites_u_idx; +CREATE INDEX group_invites_u_idx ON group_invites(user_id); +DROP INDEX groups_rooms_g_idx; +CREATE UNIQUE INDEX group_rooms_g_idx ON group_rooms(group_id, room_id); +DROP INDEX groups_rooms_r_idx; +CREATE INDEX group_rooms_r_idx ON group_rooms(room_id); +""" + +def run_create(cur, database_engine, *args, **kwargs): + rowid = "ctid" if isinstance(database_engine, PostgresEngine) then "rowid" + + # remove duplicates from group_users & group_invites tables + cur.execute(""" + DELETE FROM group_users WHERE %s NOT IN ( + SELECT min(%s) FROM group_users GROUP BY group_id, user_id + ); + """ % (rowid, rowid)); + cur.execute(""" + DELETE FROM group_invites WHERE %s NOT IN ( + SELECT min(%s) FROM group_invites GROUP BY group_id, user_id + ); + """ % (rowid, rowid)); + + for statement in get_statements(FIX_INDEXES.splitlines()): + cur.execute(statement) + +def run_upgrade(*args, **kwargs): + pass diff --git a/synapse/storage/schema/delta/48/group_unique_indexes.sql b/synapse/storage/schema/delta/48/group_unique_indexes.sql deleted file mode 100644 index 9ea7a8f2e6..0000000000 --- a/synapse/storage/schema/delta/48/group_unique_indexes.sql +++ /dev/null @@ -1,34 +0,0 @@ -/* Copyright 2018 New Vector Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - --- remove duplicates from group_users table -DELETE FROM group_users WHERE ctid NOT IN ( - SELECT min(ctid) FROM group_users GROUP BY group_id, user_id -); -DROP INDEX groups_users_g_idx; -CREATE UNIQUE INDEX group_users_g_idx ON group_users(group_id, user_id); - --- remove duplicates from group_invites table -DELETE FROM group_invites WHERE ctid NOT IN ( - SELECT min(ctid) FROM group_invites GROUP BY group_id, user_id -); -DROP INDEX groups_invites_g_idx; -CREATE UNIQUE INDEX group_invites_g_idx ON group_invites(group_id, user_id); - --- rename other indexes to actually match their table names... -ALTER INDEX groups_users_u_idx RENAME TO group_users_u_idx; -ALTER INDEX groups_invites_u_idx RENAME TO group_invites_u_idx; -ALTER INDEX groups_rooms_g_idx RENAME TO group_rooms_g_idx; -ALTER INDEX groups_rooms_r_idx RENAME TO group_rooms_r_idx; From 006e18b6bbca0721e4de957e641ef64a2f0903cf Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 28 Apr 2018 22:32:24 +0100 Subject: [PATCH 075/310] pep8 --- synapse/storage/schema/delta/48/group_unique_indexes.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/synapse/storage/schema/delta/48/group_unique_indexes.py b/synapse/storage/schema/delta/48/group_unique_indexes.py index c1f5881b3e..b653a987b1 100644 --- a/synapse/storage/schema/delta/48/group_unique_indexes.py +++ b/synapse/storage/schema/delta/48/group_unique_indexes.py @@ -32,23 +32,25 @@ DROP INDEX groups_rooms_r_idx; CREATE INDEX group_rooms_r_idx ON group_rooms(room_id); """ + def run_create(cur, database_engine, *args, **kwargs): - rowid = "ctid" if isinstance(database_engine, PostgresEngine) then "rowid" + rowid = "ctid" if isinstance(database_engine, PostgresEngine) else "rowid" # remove duplicates from group_users & group_invites tables cur.execute(""" DELETE FROM group_users WHERE %s NOT IN ( SELECT min(%s) FROM group_users GROUP BY group_id, user_id ); - """ % (rowid, rowid)); + """ % (rowid, rowid)) cur.execute(""" DELETE FROM group_invites WHERE %s NOT IN ( SELECT min(%s) FROM group_invites GROUP BY group_id, user_id ); - """ % (rowid, rowid)); + """ % (rowid, rowid)) for statement in get_statements(FIX_INDEXES.splitlines()): cur.execute(statement) + def run_upgrade(*args, **kwargs): pass From adaf3ec87fe59c59ad533cab99b5f2b1d455ab08 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 28 Apr 2018 22:39:15 +0100 Subject: [PATCH 076/310] fix missing import --- synapse/storage/schema/delta/48/group_unique_indexes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/storage/schema/delta/48/group_unique_indexes.py b/synapse/storage/schema/delta/48/group_unique_indexes.py index b653a987b1..2233af87d7 100644 --- a/synapse/storage/schema/delta/48/group_unique_indexes.py +++ b/synapse/storage/schema/delta/48/group_unique_indexes.py @@ -13,6 +13,7 @@ # limitations under the License. from synapse.storage.engines import PostgresEngine +from synapse.storage.prepare_database import get_statements FIX_INDEXES = """ -- rebuild indexes as uniques From e9143b659352e87fd9e26c8e5a771c78011bc945 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Sat, 28 Apr 2018 23:56:59 +0200 Subject: [PATCH 077/310] more bytes strings Signed-off-by: Adrian Tschira --- synapse/http/endpoint.py | 2 +- synapse/http/server.py | 2 +- synapse/rest/media/v1/upload_resource.py | 6 +++--- synapse/util/httpresourcetree.py | 7 +++++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/synapse/http/endpoint.py b/synapse/http/endpoint.py index 00572c2897..db455e5909 100644 --- a/synapse/http/endpoint.py +++ b/synapse/http/endpoint.py @@ -286,7 +286,7 @@ def resolve_service(service_name, dns_client=client, cache=SERVER_CACHE, clock=t if (len(answers) == 1 and answers[0].type == dns.SRV and answers[0].payload - and answers[0].payload.target == dns.Name('.')): + and answers[0].payload.target == dns.Name(b'.')): raise ConnectError("Service %s unavailable" % service_name) for answer in answers: diff --git a/synapse/http/server.py b/synapse/http/server.py index 8d632290de..55b9ad5251 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -546,6 +546,6 @@ def _request_user_agent_is_curl(request): b"User-Agent", default=[] ) for user_agent in user_agents: - if "curl" in user_agent: + if b"curl" in user_agent: return True return False diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index f6f498cdc5..a31e75cb46 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -81,15 +81,15 @@ class UploadResource(Resource): headers = request.requestHeaders if headers.hasHeader("Content-Type"): - media_type = headers.getRawHeaders("Content-Type")[0] + media_type = headers.getRawHeaders(b"Content-Type")[0] else: raise SynapseError( msg="Upload request missing 'Content-Type'", code=400, ) - # if headers.hasHeader("Content-Disposition"): - # disposition = headers.getRawHeaders("Content-Disposition")[0] + # if headers.hasHeader(b"Content-Disposition"): + # disposition = headers.getRawHeaders(b"Content-Disposition")[0] # TODO(markjh): parse content-dispostion content_uri = yield self.media_repo.create_content( diff --git a/synapse/util/httpresourcetree.py b/synapse/util/httpresourcetree.py index d747849553..e9f0f292ee 100644 --- a/synapse/util/httpresourcetree.py +++ b/synapse/util/httpresourcetree.py @@ -40,9 +40,12 @@ def create_resource_tree(desired_tree, root_resource): # extra resources to existing nodes. See self._resource_id for the key. resource_mappings = {} for full_path, res in desired_tree.items(): + # twisted requires all resources to be bytes + full_path = full_path.encode("utf-8") + logger.info("Attaching %s to path %s", res, full_path) last_resource = root_resource - for path_seg in full_path.split('/')[1:-1]: + for path_seg in full_path.split(b'/')[1:-1]: if path_seg not in last_resource.listNames(): # resource doesn't exist, so make a "dummy resource" child_resource = NoResource() @@ -57,7 +60,7 @@ def create_resource_tree(desired_tree, root_resource): # =========================== # now attach the actual desired resource - last_path_seg = full_path.split('/')[-1] + last_path_seg = full_path.split(b'/')[-1] # if there is already a resource here, thieve its children and # replace it From 122593265b4a10732b2e7fcfe2f5b2eba83d61d7 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Sat, 28 Apr 2018 23:53:07 +0200 Subject: [PATCH 078/310] Construct HMAC as bytes on py3 Signed-off-by: Adrian Tschira --- synapse/rest/client/v1/register.py | 16 +++++++++------- synapse/rest/client/v2_alpha/register.py | 10 ++++++---- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py index 8a82097178..9b3022e0b0 100644 --- a/synapse/rest/client/v1/register.py +++ b/synapse/rest/client/v1/register.py @@ -30,6 +30,8 @@ from hashlib import sha1 import hmac import logging +from six import string_types + logger = logging.getLogger(__name__) @@ -333,11 +335,11 @@ class RegisterRestServlet(ClientV1RestServlet): def _do_shared_secret(self, request, register_json, session): yield run_on_reactor() - if not isinstance(register_json.get("mac", None), basestring): + if not isinstance(register_json.get("mac", None), string_types): raise SynapseError(400, "Expected mac.") - if not isinstance(register_json.get("user", None), basestring): + if not isinstance(register_json.get("user", None), string_types): raise SynapseError(400, "Expected 'user' key.") - if not isinstance(register_json.get("password", None), basestring): + if not isinstance(register_json.get("password", None), string_types): raise SynapseError(400, "Expected 'password' key.") if not self.hs.config.registration_shared_secret: @@ -358,14 +360,14 @@ class RegisterRestServlet(ClientV1RestServlet): got_mac = str(register_json["mac"]) want_mac = hmac.new( - key=self.hs.config.registration_shared_secret, + key=self.hs.config.registration_shared_secret.encode(), digestmod=sha1, ) want_mac.update(user) - want_mac.update("\x00") + want_mac.update(b"\x00") want_mac.update(password) - want_mac.update("\x00") - want_mac.update("admin" if admin else "notadmin") + want_mac.update(b"\x00") + want_mac.update(b"admin" if admin else b"notadmin") want_mac = want_mac.hexdigest() if compare_digest(want_mac, got_mac): diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index f317c919dc..5cab00aea9 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -35,6 +35,8 @@ from hashlib import sha1 from synapse.util.async import run_on_reactor from synapse.util.ratelimitutils import FederationRateLimiter +from six import string_types + # We ought to be using hmac.compare_digest() but on older pythons it doesn't # exist. It's a _really minor_ security flaw to use plain string comparison @@ -210,14 +212,14 @@ class RegisterRestServlet(RestServlet): # in sessions. Pull out the username/password provided to us. desired_password = None if 'password' in body: - if (not isinstance(body['password'], basestring) or + if (not isinstance(body['password'], string_types) or len(body['password']) > 512): raise SynapseError(400, "Invalid password") desired_password = body["password"] desired_username = None if 'username' in body: - if (not isinstance(body['username'], basestring) or + if (not isinstance(body['username'], string_types) or len(body['username']) > 512): raise SynapseError(400, "Invalid username") desired_username = body['username'] @@ -243,7 +245,7 @@ class RegisterRestServlet(RestServlet): access_token = get_access_token_from_request(request) - if isinstance(desired_username, basestring): + if isinstance(desired_username, string_types): result = yield self._do_appservice_registration( desired_username, access_token, body ) @@ -464,7 +466,7 @@ class RegisterRestServlet(RestServlet): # includes the password and admin flag in the hashed text. Why are # these different? want_mac = hmac.new( - key=self.hs.config.registration_shared_secret, + key=self.hs.config.registration_shared_secret.encode(), msg=user, digestmod=sha1, ).hexdigest() From 7767a9fc0e2adbe1828e0bd86e83446a9fb66bc7 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 30 Apr 2018 00:37:32 +0100 Subject: [PATCH 079/310] Update tox.ini add missing comma --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index c6cee92568..ca8373662b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = packaging, py27, py36 pep8 +envlist = packaging, py27, py36, pep8 [testenv] deps = From af3cc5051124982182e7151e35fb72f857d81392 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Apr 2018 12:09:47 +0100 Subject: [PATCH 080/310] Remove redundant call to preserve_fn submit_event_for_as doesn't return a deferred anyway, so this is pointless. --- synapse/handlers/appservice.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 6cc2388306..f6c7b89270 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -19,7 +19,7 @@ import synapse from synapse.api.constants import EventTypes from synapse.util.metrics import Measure from synapse.util.logcontext import ( - make_deferred_yieldable, preserve_fn, run_in_background, + make_deferred_yieldable, run_in_background, ) import logging @@ -111,9 +111,7 @@ class ApplicationServicesHandler(object): # Fork off pushes to these services for service in services: - preserve_fn(self.scheduler.submit_event_for_as)( - service, event - ) + self.scheduler.submit_event_for_as(service, event) @defer.inlineCallbacks def handle_room_events(events): From d1d54d60889f94228adff55f64116aa66d07ac3a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 30 Apr 2018 00:58:31 +0100 Subject: [PATCH 081/310] add py36 to build matrix --- .travis.yml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3ce93cb434..e6ba6f4752 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,22 @@ sudo: false language: python -python: 2.7 # tell travis to cache ~/.cache/pip cache: pip -env: - - TOX_ENV=packaging - - TOX_ENV=pep8 - - TOX_ENV=py27 +matrix: + include: + - python: 2.7 + env: TOX_ENV=packaging + + - python: 2.7 + env: TOX_ENV=pep8 + + - python: 2.7 + env: TOX_ENV=py27 + + - python: 3.6 + env: TOX_ENV=py36 install: - pip install tox From 0c9db26260210bd2066048333b2644a2511b1801 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Mon, 30 Apr 2018 09:49:10 +0200 Subject: [PATCH 082/310] add comment explaining attributeerror --- synapse/events/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index d4d1b92f7a..c3ff85c49a 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -47,6 +47,9 @@ class _EventInternalMetadata(object): def _event_dict_property(key): + # We want to be able to use hasattr with the event dict properties. + # However, (on python3) hasattr expects AttributeError to be raised. Hence, + # we need to transform the KeyError into an AttributeError def getter(self): try: return self._event_dict[key] From 6e005d13821b86e4727f4f28f4c7b1e51c05d096 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Mon, 30 Apr 2018 10:37:41 +0200 Subject: [PATCH 083/310] run config tests on py3 Signed-off-by: Adrian Tschira --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ca8373662b..89ca6d7a19 100644 --- a/tox.ini +++ b/tox.ini @@ -51,7 +51,8 @@ usedevelop=true commands = /usr/bin/find "{toxinidir}" -name '*.pyc' -delete coverage run {env:COVERAGE_OPTS:} --source="{toxinidir}/synapse" \ - "{envbindir}/trial" {env:TRIAL_FLAGS:} {posargs:tests/metrics} {env:TOXSUFFIX:} + "{envbindir}/trial" {env:TRIAL_FLAGS:} {posargs:tests/metrics tests/config} \ + {env:TOXSUFFIX:} {env:DUMP_COVERAGE_COMMAND:coverage report -m} [testenv:packaging] From 576b71dd3d7139ae246b900fea4533d813f38a2a Mon Sep 17 00:00:00 2001 From: Krombel Date: Mon, 30 Apr 2018 14:29:48 +0200 Subject: [PATCH 084/310] add guard for None on purge_history api --- synapse/rest/client/v1/admin.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index 303419d281..efd5c9873d 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -168,11 +168,24 @@ class PurgeHistoryRestServlet(ClientV1RestServlet): yield self.store.find_first_stream_ordering_after_ts(ts) ) - (_, depth, _) = ( + room_event_after_stream_ordering = ( yield self.store.get_room_event_after_stream_ordering( room_id, stream_ordering, ) ) + if room_event_after_stream_ordering: + (_, depth, _) = room_event_after_stream_ordering + else: + logger.warn( + "[purge] purging events not possible: No event found " + "(received_ts %i => stream_ordering %i)", + ts, stream_ordering, + ) + raise SynapseError( + 404, + "there is no event to be purged", + errcode=Codes.NOT_FOUND, + ) logger.info( "[purge] purging up to depth %i (received_ts %i => " "stream_ordering %i)", From 2ad3fc36e66fcc86cc3fde99760e4851402a8d56 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Mon, 30 Apr 2018 16:21:11 +0100 Subject: [PATCH 085/310] Fixes #3135 - Replace _OpenSSLECCurve with crypto.get_elliptic_curve (#3157) fixes #3135 Signed-off-by: Will Hunt will@half-shot.uk --- synapse/crypto/context_factory.py | 9 +++++---- synapse/python_dependencies.py | 7 +++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index cff3ca809a..0397f73ab4 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -13,8 +13,8 @@ # limitations under the License. from twisted.internet import ssl -from OpenSSL import SSL -from twisted.internet._sslverify import _OpenSSLECCurve, _defaultCurveName +from OpenSSL import SSL, crypto +from twisted.internet._sslverify import _defaultCurveName import logging @@ -32,8 +32,9 @@ class ServerContextFactory(ssl.ContextFactory): @staticmethod def configure_context(context, config): try: - _ecCurve = _OpenSSLECCurve(_defaultCurveName) - _ecCurve.addECKeyToContext(context) + _ecCurve = crypto.get_elliptic_curve(_defaultCurveName) + context.set_tmp_ecdh(_ecCurve) + except Exception: logger.exception("Failed to enable elliptic curve for TLS") context.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 711cbb6c50..216db4d164 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -39,12 +39,11 @@ REQUIREMENTS = { "signedjson>=1.0.0": ["signedjson>=1.0.0"], "pynacl>=1.2.1": ["nacl>=1.2.1", "nacl.bindings"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], + "Twisted>=16.0.0": ["twisted>=16.0.0"], - # we break under Twisted 18.4 - # (https://github.com/matrix-org/synapse/issues/3135) - "Twisted>=16.0.0,<18.4": ["twisted>=16.0.0"], + # We use crypto.get_elliptic_curve which is only supported in >=0.15 + "pyopenssl>=0.15": ["OpenSSL>=0.15"], - "pyopenssl>=0.14": ["OpenSSL>=0.14"], "pyyaml": ["yaml"], "pyasn1": ["pyasn1"], "daemonize": ["daemonize"], From 6495dbb326dd2b5d58e5de25107f7fe6d13b6ca4 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Mon, 30 Apr 2018 21:58:30 +0200 Subject: [PATCH 086/310] Burminate v1auth This closes #2602 v1auth was created to account for the differences in status code between the v1 and v2_alpha revisions of the protocol (401 vs 403 for invalid tokens). However since those protocols were merged, this makes the r0 version/endpoint internally inconsistent, and violates the specification for the r0 endpoint. This might break clients that rely on this inconsistency with the specification. This is said to affect the legacy angular reference client. However, I feel that restoring parity with the spec is more important. Either way, it is critical to inform developers about this change, in case they rely on the illegal behaviour. Signed-off-by: Adrian Tschira --- synapse/rest/client/v1/base.py | 6 +++++- synapse/rest/client/v1/pusher.py | 2 +- synapse/server.py | 10 ---------- tests/rest/client/v1/test_events.py | 9 +++++++-- tests/rest/client/v1/test_profile.py | 2 +- tests/rest/client/v1/test_rooms.py | 18 +++++++++--------- tests/rest/client/v1/test_typing.py | 2 +- 7 files changed, 24 insertions(+), 25 deletions(-) diff --git a/synapse/rest/client/v1/base.py b/synapse/rest/client/v1/base.py index c7aa0bbf59..197335d7aa 100644 --- a/synapse/rest/client/v1/base.py +++ b/synapse/rest/client/v1/base.py @@ -52,6 +52,10 @@ class ClientV1RestServlet(RestServlet): """A base Synapse REST Servlet for the client version 1 API. """ + # This subclass was presumably created to allow the auth for the v1 + # protocol version to be different, however this behaviour was removed. + # it may no longer be necessary + def __init__(self, hs): """ Args: @@ -59,5 +63,5 @@ class ClientV1RestServlet(RestServlet): """ self.hs = hs self.builder_factory = hs.get_event_builder_factory() - self.auth = hs.get_v1auth() + self.auth = hs.get_auth() self.txns = HttpTransactionCache(hs.get_clock()) diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 1819a560cb..0206e664c1 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -150,7 +150,7 @@ class PushersRemoveRestServlet(RestServlet): super(RestServlet, self).__init__() self.hs = hs self.notifier = hs.get_notifier() - self.auth = hs.get_v1auth() + self.auth = hs.get_auth() self.pusher_pool = self.hs.get_pusherpool() @defer.inlineCallbacks diff --git a/synapse/server.py b/synapse/server.py index cd0c1a51be..ebdea6b0c4 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -105,7 +105,6 @@ class HomeServer(object): 'federation_client', 'federation_server', 'handlers', - 'v1auth', 'auth', 'state_handler', 'state_resolution_handler', @@ -225,15 +224,6 @@ class HomeServer(object): def build_simple_http_client(self): return SimpleHttpClient(self) - def build_v1auth(self): - orf = Auth(self) - # Matrix spec makes no reference to what HTTP status code is returned, - # but the V1 API uses 403 where it means 401, and the webclient - # relies on this behaviour, so V1 gets its own copy of the auth - # with backwards compat behaviour. - orf.TOKEN_NOT_FOUND_HTTP_STATUS = 403 - return orf - def build_state_handler(self): return StateHandler(self) diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py index a8d09600bd..f5a7258e68 100644 --- a/tests/rest/client/v1/test_events.py +++ b/tests/rest/client/v1/test_events.py @@ -148,11 +148,16 @@ class EventStreamPermissionsTestCase(RestTestCase): @defer.inlineCallbacks def test_stream_basic_permissions(self): - # invalid token, expect 403 + # invalid token, expect 401 + # note: this is in violation of the original v1 spec, which expected + # 403. However, since the v1 spec no longer exists and the v1 + # implementation is now part of the r0 implementation, the newer + # behaviour is used instead to be consistent with the r0 spec. + # see issue #2602 (code, response) = yield self.mock_resource.trigger_get( "/events?access_token=%s" % ("invalid" + self.token, ) ) - self.assertEquals(403, code, msg=str(response)) + self.assertEquals(401, code, msg=str(response)) # valid token, expect content (code, response) = yield self.mock_resource.trigger_get( diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py index deac7f100c..dc94b8bd19 100644 --- a/tests/rest/client/v1/test_profile.py +++ b/tests/rest/client/v1/test_profile.py @@ -52,7 +52,7 @@ class ProfileTestCase(unittest.TestCase): def _get_user_by_req(request=None, allow_guest=False): return synapse.types.create_requester(myid) - hs.get_v1auth().get_user_by_req = _get_user_by_req + hs.get_auth().get_user_by_req = _get_user_by_req profile.register_servlets(hs, self.mock_resource) diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index d763400eaf..61d737725b 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -60,7 +60,7 @@ class RoomPermissionsTestCase(RestTestCase): "token_id": 1, "is_guest": False, } - hs.get_v1auth().get_user_by_access_token = get_user_by_access_token + hs.get_auth().get_user_by_access_token = get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -70,7 +70,7 @@ class RoomPermissionsTestCase(RestTestCase): synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) - self.auth = hs.get_v1auth() + self.auth = hs.get_auth() # create some rooms under the name rmcreator_id self.uncreated_rmid = "!aa:test" @@ -425,7 +425,7 @@ class RoomsMemberListTestCase(RestTestCase): "token_id": 1, "is_guest": False, } - hs.get_v1auth().get_user_by_access_token = get_user_by_access_token + hs.get_auth().get_user_by_access_token = get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -507,7 +507,7 @@ class RoomsCreateTestCase(RestTestCase): "token_id": 1, "is_guest": False, } - hs.get_v1auth().get_user_by_access_token = get_user_by_access_token + hs.get_auth().get_user_by_access_token = get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -597,7 +597,7 @@ class RoomTopicTestCase(RestTestCase): "is_guest": False, } - hs.get_v1auth().get_user_by_access_token = get_user_by_access_token + hs.get_auth().get_user_by_access_token = get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -711,7 +711,7 @@ class RoomMemberStateTestCase(RestTestCase): "token_id": 1, "is_guest": False, } - hs.get_v1auth().get_user_by_access_token = get_user_by_access_token + hs.get_auth().get_user_by_access_token = get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -843,7 +843,7 @@ class RoomMessagesTestCase(RestTestCase): "token_id": 1, "is_guest": False, } - hs.get_v1auth().get_user_by_access_token = get_user_by_access_token + hs.get_auth().get_user_by_access_token = get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -945,7 +945,7 @@ class RoomInitialSyncTestCase(RestTestCase): "token_id": 1, "is_guest": False, } - hs.get_v1auth().get_user_by_access_token = get_user_by_access_token + hs.get_auth().get_user_by_access_token = get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -1017,7 +1017,7 @@ class RoomMessageListTestCase(RestTestCase): "token_id": 1, "is_guest": False, } - hs.get_v1auth().get_user_by_access_token = get_user_by_access_token + hs.get_auth().get_user_by_access_token = get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index 2ec4ecab5b..fe161ee5cb 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -68,7 +68,7 @@ class RoomTypingTestCase(RestTestCase): "is_guest": False, } - hs.get_v1auth().get_user_by_access_token = get_user_by_access_token + hs.get_auth().get_user_by_access_token = get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) From 5917562b6040448cd26b67fac52d61b41ba08f6d Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Tue, 1 May 2018 12:12:22 +0100 Subject: [PATCH 087/310] 10 mins seems more reasonable that every minute --- synapse/app/homeserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 808567365a..328ab8032c 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -488,7 +488,7 @@ def run(hs): # be quite busy the first few minutes clock.call_later(5 * 60, phone_stats_home) - clock.looping_call(generate_user_daily_visit_stats, 60 * 1000) + clock.looping_call(generate_user_daily_visit_stats, 10 * 60 * 1000) clock.call_later(5 * 60, generate_user_daily_visit_stats) if hs.config.daemonize and hs.config.print_pidfile: From d0857702e8e22b8bd3dae7fc3cd3718da1e94f19 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Tue, 1 May 2018 12:12:57 +0100 Subject: [PATCH 088/310] add inidexes based on usage --- synapse/storage/schema/delta/49/add_user_daily_visits.sql | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/synapse/storage/schema/delta/49/add_user_daily_visits.sql b/synapse/storage/schema/delta/49/add_user_daily_visits.sql index 11356a97fc..3dd478196f 100644 --- a/synapse/storage/schema/delta/49/add_user_daily_visits.sql +++ b/synapse/storage/schema/delta/49/add_user_daily_visits.sql @@ -16,10 +16,6 @@ CREATE TABLE user_daily_visits ( user_id TEXT NOT NULL, device_id TEXT, - user_agent TEXT NOT NULL, timestamp BIGINT NOT NULL ); - -/* What indexes should I include? - * Reads are offline so should optimise for writes - * Need to check if already an entry so user,day - */ +CREATE INDEX user_daily_visits_uts_idx ON user_daily_visits(user_id, timestamp); +CREATE INDEX user_daily_visits_ts_idx ON user_daily_visits(timestamp); From dd1a8324195f37d508d518db10ba37d9028afc86 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Tue, 1 May 2018 12:13:49 +0100 Subject: [PATCH 089/310] remove user agent from data model, will just join on user_ips --- synapse/storage/__init__.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index c22d38800c..b51cf70336 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -15,6 +15,7 @@ # limitations under the License. import datetime +from dateutil import tz import time import logging @@ -354,10 +355,9 @@ class DataStore(RoomMemberStore, RoomStore, def _generate_user_daily_visits(txn): logger.info("Calling _generate_user_daily_visits") # determine timestamp of previous days - yesterday = datetime.datetime.now() - datetime.timedelta(days=1) - yesterday_start = datetime.datetime(yesterday.year, - yesterday.month, - yesterday.day, 0, 0, 0, 0) + yesterday = datetime.datetime.utcnow() - datetime.timedelta(days=1) + yesterday_start = datetime.datetime(yesterday.year, yesterday.month, + yesterday.day, tzinfo=tz.tzutc()) yesterday_start_time = int(time.mktime(yesterday_start.timetuple())) * 1000 # Check that this job has not already been completed @@ -371,9 +371,8 @@ class DataStore(RoomMemberStore, RoomStore, # Bail if the most recent time is yesterday if row and row[0] == yesterday_start_time: - logger.info("Bailing from _generate_user_daily_visits, already completed") return - logger.info("inserting into user_daily_visits") + # Not specificying an upper bound means that if the update is run at # 10 mins past midnight and the user is active during a 30 min session # that the user is still included in the previous days stats @@ -382,20 +381,20 @@ class DataStore(RoomMemberStore, RoomStore, # The alternative is to insert on every request - but prefer to avoid # for performance reasons sql = """ - SELECT user_id, user_agent, device_id + SELECT user_id, device_id FROM user_ips WHERE last_seen > ? """ txn.execute(sql, (yesterday_start_time,)) + user_visits = txn.fetchall() sql = """ - INSERT INTO user_daily_visits (user_id. user_agent, - device_id, timestamp) - VALUES (?, ?, ?, ?) + INSERT INTO user_daily_visits (user_id, device_id, timestamp) + VALUES (?, ?, ?) """ - for row in txn: - txn.execute(sql, (row + (yesterday_start_time,))) + for visit in user_visits: + txn.execute(sql, (visit + (yesterday_start_time,))) return self.runInteraction("generate_user_daily_visits", _generate_user_daily_visits) From 9f21de6a015a210d9ce5ae71948af98a56046ea2 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 1 May 2018 19:19:46 +0100 Subject: [PATCH 090/310] missing word :| --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0dd2efd9d2..317846d2a2 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -10,7 +10,7 @@ SECURITY UPDATE of events in such a room will default to using stream_ordering rather than depth (topological_ordering). - This is a temporary solution to mitigate abuse in the wild, whilst a long solution + This is a temporary solution to mitigate abuse in the wild, whilst a long term solution is being implemented to improve how the depth parameter is used. Full details at From d4c14e143825c679a7909b009aec2e51f829857f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Tue, 1 May 2018 20:47:58 +0200 Subject: [PATCH 091/310] Fix the documentation about 'POSTGRES_DB' --- contrib/docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 25c358c847..aed56646c2 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -135,7 +135,7 @@ Shared secrets, that will be initialized to random values if not set: Database specific values (will use SQLite if not set): -* `POSTGRES_DATABASE` - The database name for the synapse postgres database. [default: `matrix`] +* `POSTGRES_DB` - The database name for the synapse postgres database. [default: `synapse`] * `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `db` which is useful when using a container on the same docker network in a compose file where the postgres service is called `db`] * `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy. * `POSTGRES_USER` - The user for the synapse postgres database. [default: `matrix`] From 4f2e898c29de5b51e8c5893afa64e5b43d25048f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Tue, 1 May 2018 20:49:52 +0200 Subject: [PATCH 092/310] Make the logging level configurable --- contrib/docker/conf/log.config | 6 +++--- contrib/docker/docker-compose.yml | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/docker/conf/log.config b/contrib/docker/conf/log.config index b5c907c4f9..1851995802 100644 --- a/contrib/docker/conf/log.config +++ b/contrib/docker/conf/log.config @@ -17,13 +17,13 @@ handlers: loggers: synapse: - level: INFO + level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }} synapse.storage.SQL: # beware: increasing this to DEBUG will make synapse log sensitive # information such as access tokens. - level: INFO + level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }} root: - level: INFO + level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }} handlers: [console] diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 46e72601d3..0b531949e0 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -15,6 +15,7 @@ services: - SYNAPSE_SERVER_NAME=my.matrix.host - SYNAPSE_REPORT_STATS=no - SYNAPSE_ENABLE_REGISTRATION=yes + - SYNAPSE_LOG_LEVEL=INFO - POSTGRES_PASSWORD=changeme volumes: # You may either store all the files in a local folder From e482f8cd8504b36dc1ce2c1e51e0dee479d33249 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 2 May 2018 09:12:26 +0100 Subject: [PATCH 093/310] Fix incorrect reference to StringIO This was introduced in 4f2f5171 --- synapse/util/logformatter.py | 2 +- tests/util/test_logformatter.py | 38 +++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 tests/util/test_logformatter.py diff --git a/synapse/util/logformatter.py b/synapse/util/logformatter.py index 59ab3c6968..3e42868ea9 100644 --- a/synapse/util/logformatter.py +++ b/synapse/util/logformatter.py @@ -32,7 +32,7 @@ class LogFormatter(logging.Formatter): super(LogFormatter, self).__init__(*args, **kwargs) def formatException(self, ei): - sio = StringIO.StringIO() + sio = StringIO() (typ, val, tb) = ei # log the stack above the exception capture point if possible, but diff --git a/tests/util/test_logformatter.py b/tests/util/test_logformatter.py new file mode 100644 index 0000000000..1a1a8412f2 --- /dev/null +++ b/tests/util/test_logformatter.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys + +from synapse.util.logformatter import LogFormatter +from tests import unittest + + +class TestException(Exception): + pass + + +class LogFormatterTestCase(unittest.TestCase): + def test_formatter(self): + formatter = LogFormatter() + + try: + raise TestException("testytest") + except TestException: + ei = sys.exc_info() + + output = formatter.formatException(ei) + + # check the output looks vaguely sane + self.assertIn("testytest", output) + self.assertIn("Capture point", output) From f22e7cda2c72d461acba664cb083e8c4e3c7572a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 2 May 2018 11:46:23 +0100 Subject: [PATCH 094/310] Fix a class of logcontext leaks So, it turns out that if you have a first `Deferred` `D1`, you can add a callback which returns another `Deferred` `D2`, and `D2` must then complete before any further callbacks on `D1` will execute (and later callbacks on `D1` get the *result* of `D2` rather than `D2` itself). So, `D1` might have `called=True` (as in, it has started running its callbacks), but any new callbacks added to `D1` won't get run until `D2` completes - so if you `yield D1` in an `inlineCallbacks` function, your `yield` will 'block'. In conclusion: some of our assumptions in `logcontext` were invalid. We need to make sure that we don't optimise out the logcontext juggling when this situation happens. Fortunately, it is easy to detect by checking `D1.paused`. --- synapse/util/logcontext.py | 58 +++++++++++++++++++----------- tests/util/test_logcontext.py | 67 +++++++++++++++++++++++++++++------ 2 files changed, 93 insertions(+), 32 deletions(-) diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index 01ac71e53e..e086e12213 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -302,7 +302,7 @@ def preserve_fn(f): def run_in_background(f, *args, **kwargs): """Calls a function, ensuring that the current context is restored after return from the function, and that the sentinel context is set once the - deferred returned by the funtion completes. + deferred returned by the function completes. Useful for wrapping functions that return a deferred which you don't yield on (for instance because you want to pass it to deferred.gatherResults()). @@ -320,24 +320,31 @@ def run_in_background(f, *args, **kwargs): # by synchronous exceptions, so let's turn them into Failures. return defer.fail() - if isinstance(res, defer.Deferred) and not res.called: - # The function will have reset the context before returning, so - # we need to restore it now. - LoggingContext.set_current_context(current) + if not isinstance(res, defer.Deferred): + return res - # The original context will be restored when the deferred - # completes, but there is nothing waiting for it, so it will - # get leaked into the reactor or some other function which - # wasn't expecting it. We therefore need to reset the context - # here. - # - # (If this feels asymmetric, consider it this way: we are - # effectively forking a new thread of execution. We are - # probably currently within a ``with LoggingContext()`` block, - # which is supposed to have a single entry and exit point. But - # by spawning off another deferred, we are effectively - # adding a new exit point.) - res.addBoth(_set_context_cb, LoggingContext.sentinel) + if res.called and not res.paused: + # The function should have maintained the logcontext, so we can + # optimise out the messing about + return res + + # The function may have reset the context before returning, so + # we need to restore it now. + ctx = LoggingContext.set_current_context(current) + + # The original context will be restored when the deferred + # completes, but there is nothing waiting for it, so it will + # get leaked into the reactor or some other function which + # wasn't expecting it. We therefore need to reset the context + # here. + # + # (If this feels asymmetric, consider it this way: we are + # effectively forking a new thread of execution. We are + # probably currently within a ``with LoggingContext()`` block, + # which is supposed to have a single entry and exit point. But + # by spawning off another deferred, we are effectively + # adding a new exit point.) + res.addBoth(_set_context_cb, ctx) return res @@ -354,9 +361,18 @@ def make_deferred_yieldable(deferred): (This is more-or-less the opposite operation to run_in_background.) """ - if isinstance(deferred, defer.Deferred) and not deferred.called: - prev_context = LoggingContext.set_current_context(LoggingContext.sentinel) - deferred.addBoth(_set_context_cb, prev_context) + if not isinstance(deferred, defer.Deferred): + return deferred + + if deferred.called and not deferred.paused: + # it looks like this deferred is ready to run any callbacks we give it + # immediately. We may as well optimise out the logcontext faffery. + return deferred + + # ok, we can't be sure that a yield won't block, so let's reset the + # logcontext, and add a callback to the deferred to restore it. + prev_context = LoggingContext.set_current_context(LoggingContext.sentinel) + deferred.addBoth(_set_context_cb, prev_context) return deferred diff --git a/tests/util/test_logcontext.py b/tests/util/test_logcontext.py index 4850722bc5..7707fbb1f0 100644 --- a/tests/util/test_logcontext.py +++ b/tests/util/test_logcontext.py @@ -36,24 +36,28 @@ class LoggingContextTestCase(unittest.TestCase): yield sleep(0) self._check_test_key("one") - def _test_preserve_fn(self, function): + def _test_run_in_background(self, function): sentinel_context = LoggingContext.current_context() callback_completed = [False] - @defer.inlineCallbacks - def cb(): + def test(): context_one.request = "one" - yield function() - self._check_test_key("one") + d = function() - callback_completed[0] = True + def cb(res): + self._check_test_key("one") + callback_completed[0] = True + return res + d.addCallback(cb) + + return d with LoggingContext() as context_one: context_one.request = "one" # fire off function, but don't wait on it. - logcontext.preserve_fn(cb)() + logcontext.run_in_background(test) self._check_test_key("one") @@ -80,20 +84,31 @@ class LoggingContextTestCase(unittest.TestCase): # test is done once d2 finishes return d2 - def test_preserve_fn_with_blocking_fn(self): + def test_run_in_background_with_blocking_fn(self): @defer.inlineCallbacks def blocking_function(): yield sleep(0) - return self._test_preserve_fn(blocking_function) + return self._test_run_in_background(blocking_function) - def test_preserve_fn_with_non_blocking_fn(self): + def test_run_in_background_with_non_blocking_fn(self): @defer.inlineCallbacks def nonblocking_function(): with logcontext.PreserveLoggingContext(): yield defer.succeed(None) - return self._test_preserve_fn(nonblocking_function) + return self._test_run_in_background(nonblocking_function) + + @unittest.DEBUG + def test_run_in_background_with_chained_deferred(self): + # a function which returns a deferred which looks like it has been + # called, but is actually paused + def testfunc(): + return logcontext.make_deferred_yieldable( + _chained_deferred_function() + ) + + return self._test_run_in_background(testfunc) @defer.inlineCallbacks def test_make_deferred_yieldable(self): @@ -118,6 +133,22 @@ class LoggingContextTestCase(unittest.TestCase): # now it should be restored self._check_test_key("one") + @defer.inlineCallbacks + def test_make_deferred_yieldable_with_chained_deferreds(self): + sentinel_context = LoggingContext.current_context() + + with LoggingContext() as context_one: + context_one.request = "one" + + d1 = logcontext.make_deferred_yieldable(_chained_deferred_function()) + # make sure that the context was reset by make_deferred_yieldable + self.assertIs(LoggingContext.current_context(), sentinel_context) + + yield d1 + + # now it should be restored + self._check_test_key("one") + @defer.inlineCallbacks def test_make_deferred_yieldable_on_non_deferred(self): """Check that make_deferred_yieldable does the right thing when its @@ -132,3 +163,17 @@ class LoggingContextTestCase(unittest.TestCase): r = yield d1 self.assertEqual(r, "bum") self._check_test_key("one") + + +# a function which returns a deferred which has been "called", but +# which had a function which returned another incomplete deferred on +# its callback list, so won't yet call any other new callbacks. +def _chained_deferred_function(): + d = defer.succeed(None) + + def cb(res): + d2 = defer.Deferred() + reactor.callLater(0, d2.callback, res) + return d2 + d.addCallback(cb) + return d From 46beeb9a307febb679fc25565aca439f8af044ed Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 2 May 2018 15:46:22 +0100 Subject: [PATCH 095/310] Fix a couple of logcontext leaks in unit tests ... which were making other, innocent, tests, fail. Plus remove a spurious unittest.DEBUG which was making the output noisy. --- tests/appservice/test_scheduler.py | 11 +++++++++-- tests/storage/test_event_push_actions.py | 1 - 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/appservice/test_scheduler.py b/tests/appservice/test_scheduler.py index e5a902f734..9181692771 100644 --- a/tests/appservice/test_scheduler.py +++ b/tests/appservice/test_scheduler.py @@ -17,6 +17,8 @@ from synapse.appservice.scheduler import ( _ServiceQueuer, _TransactionController, _Recoverer ) from twisted.internet import defer + +from synapse.util.logcontext import make_deferred_yieldable from ..utils import MockClock from mock import Mock from tests import unittest @@ -204,7 +206,9 @@ class ApplicationServiceSchedulerQueuerTestCase(unittest.TestCase): def test_send_single_event_with_queue(self): d = defer.Deferred() - self.txn_ctrl.send = Mock(return_value=d) + self.txn_ctrl.send = Mock( + side_effect=lambda x, y: make_deferred_yieldable(d), + ) service = Mock(id=4) event = Mock(event_id="first") event2 = Mock(event_id="second") @@ -235,7 +239,10 @@ class ApplicationServiceSchedulerQueuerTestCase(unittest.TestCase): srv_2_event2 = Mock(event_id="srv2b") send_return_list = [srv_1_defer, srv_2_defer] - self.txn_ctrl.send = Mock(side_effect=lambda x, y: send_return_list.pop(0)) + + def do_send(x, y): + return make_deferred_yieldable(send_return_list.pop(0)) + self.txn_ctrl.send = Mock(side_effect=do_send) # send events for different ASes and make sure they are sent self.queuer.enqueue(srv1, srv_1_event) diff --git a/tests/storage/test_event_push_actions.py b/tests/storage/test_event_push_actions.py index 575374c6a6..9962ce8a5d 100644 --- a/tests/storage/test_event_push_actions.py +++ b/tests/storage/test_event_push_actions.py @@ -128,7 +128,6 @@ class EventPushActionsStoreTestCase(tests.unittest.TestCase): yield _rotate(10) yield _assert_counts(1, 1) - @tests.unittest.DEBUG @defer.inlineCallbacks def test_find_first_stream_ordering_after_ts(self): def add_event(so, ts): From 11607006d9570f76a46b44c9c8de13f13550dbcf Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 2 May 2018 15:48:47 +0100 Subject: [PATCH 096/310] Remove spurious unittest.DEBUG --- tests/util/test_logcontext.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/util/test_logcontext.py b/tests/util/test_logcontext.py index 7707fbb1f0..ad78d884e0 100644 --- a/tests/util/test_logcontext.py +++ b/tests/util/test_logcontext.py @@ -99,7 +99,6 @@ class LoggingContextTestCase(unittest.TestCase): return self._test_run_in_background(nonblocking_function) - @unittest.DEBUG def test_run_in_background_with_chained_deferred(self): # a function which returns a deferred which looks like it has been # called, but is actually paused From be31adb036cab08c4c70cb1e0d0f6550f0a75ef1 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 2 May 2018 13:29:16 +0100 Subject: [PATCH 097/310] Fix logcontext leak in media repo Make FileResponder.write_to_consumer uphold the logcontext contract --- synapse/rest/media/v1/_base.py | 1 + synapse/rest/media/v1/media_storage.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/rest/media/v1/_base.py b/synapse/rest/media/v1/_base.py index d9c4af9389..c0d2f06855 100644 --- a/synapse/rest/media/v1/_base.py +++ b/synapse/rest/media/v1/_base.py @@ -143,6 +143,7 @@ def respond_with_responder(request, responder, media_type, file_size, upload_nam respond_404(request) return + logger.debug("Responding to media request with responder %s") add_file_headers(request, media_type, file_size, upload_name) with responder: yield responder.write_to_consumer(request) diff --git a/synapse/rest/media/v1/media_storage.py b/synapse/rest/media/v1/media_storage.py index 7f263db239..d23fe10b07 100644 --- a/synapse/rest/media/v1/media_storage.py +++ b/synapse/rest/media/v1/media_storage.py @@ -255,7 +255,9 @@ class FileResponder(Responder): self.open_file = open_file def write_to_consumer(self, consumer): - return FileSender().beginFileTransfer(self.open_file, consumer) + return make_deferred_yieldable( + FileSender().beginFileTransfer(self.open_file, consumer) + ) def __exit__(self, exc_type, exc_val, exc_tb): self.open_file.close() From 32015e1109bc955697353d8f8088e3f6b538d12c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 2 May 2018 16:52:42 +0100 Subject: [PATCH 098/310] Escape label values in prometheus metrics --- synapse/metrics/metric.py | 22 ++++++++++++++++++++-- tests/metrics/test_metric.py | 21 ++++++++++++++++++++- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/synapse/metrics/metric.py b/synapse/metrics/metric.py index 89bd47c3f7..1a09e417c9 100644 --- a/synapse/metrics/metric.py +++ b/synapse/metrics/metric.py @@ -16,6 +16,7 @@ from itertools import chain import logging +import re logger = logging.getLogger(__name__) @@ -56,8 +57,7 @@ class BaseMetric(object): return not len(self.labels) def _render_labelvalue(self, value): - # TODO: escape backslashes, quotes and newlines - return '"%s"' % (value) + return '"%s"' % (_escape_label_value(value),) def _render_key(self, values): if self.is_scalar(): @@ -299,3 +299,21 @@ class MemoryUsageMetric(object): "process_psutil_rss:total %d" % sum_rss, "process_psutil_rss:count %d" % len_rss, ] + + +def _escape_character(c): + """Replaces a single character with its escape sequence. + """ + if c == "\\": + return "\\\\" + elif c == "\"": + return "\\\"" + elif c == "\n": + return "\\n" + return c + + +def _escape_label_value(value): + """Takes a label value and escapes quotes, newlines and backslashes + """ + return re.sub(r"([\n\"\\])", lambda m: _escape_character(m.group(1)), value) diff --git a/tests/metrics/test_metric.py b/tests/metrics/test_metric.py index 39bde6e3f8..069c0be762 100644 --- a/tests/metrics/test_metric.py +++ b/tests/metrics/test_metric.py @@ -16,7 +16,8 @@ from tests import unittest from synapse.metrics.metric import ( - CounterMetric, CallbackMetric, DistributionMetric, CacheMetric + CounterMetric, CallbackMetric, DistributionMetric, CacheMetric, + _escape_label_value, ) @@ -171,3 +172,21 @@ class CacheMetricTestCase(unittest.TestCase): 'cache:size{name="cache_name"} 1', 'cache:evicted_size{name="cache_name"} 2', ]) + + +class LabelValueEscapeTestCase(unittest.TestCase): + def test_simple(self): + string = "safjhsdlifhyskljfksdfh" + self.assertEqual(string, _escape_label_value(string)) + + def test_escape(self): + self.assertEqual( + "abc\\\"def\\nghi\\\\", + _escape_label_value("abc\"def\nghi\\"), + ) + + def test_sequence_of_escapes(self): + self.assertEqual( + "abc\\\"def\\nghi\\\\\\n", + _escape_label_value("abc\"def\nghi\\\n"), + ) From a41117c63b4e7ef2021fb59cb521c77d72f2bf70 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 2 May 2018 17:27:27 +0100 Subject: [PATCH 099/310] Make _escape_character take MatchObject --- synapse/metrics/metric.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/synapse/metrics/metric.py b/synapse/metrics/metric.py index 1a09e417c9..fbba94e633 100644 --- a/synapse/metrics/metric.py +++ b/synapse/metrics/metric.py @@ -301,9 +301,17 @@ class MemoryUsageMetric(object): ] -def _escape_character(c): +def _escape_character(m): """Replaces a single character with its escape sequence. + + Args: + m (re.MatchObject): A match object whose first group is the single + character to replace + + Returns: + str """ + c = m.group(1) if c == "\\": return "\\\\" elif c == "\"": @@ -316,4 +324,4 @@ def _escape_character(c): def _escape_label_value(value): """Takes a label value and escapes quotes, newlines and backslashes """ - return re.sub(r"([\n\"\\])", lambda m: _escape_character(m.group(1)), value) + return re.sub(r"([\n\"\\])", _escape_character, value) From 1dfd650348251508ac044b3db114092b1e857a74 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 2 May 2018 22:42:36 +0100 Subject: [PATCH 100/310] add missing param to cancelled_to_request_timed_out_error This gets two arguments, not one. --- synapse/http/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/__init__.py b/synapse/http/__init__.py index 0d47ccdb59..054372e179 100644 --- a/synapse/http/__init__.py +++ b/synapse/http/__init__.py @@ -25,7 +25,7 @@ class RequestTimedOutError(SynapseError): super(RequestTimedOutError, self).__init__(504, "Timed out") -def cancelled_to_request_timed_out_error(value): +def cancelled_to_request_timed_out_error(value, timeout): """Turns CancelledErrors into RequestTimedOutErrors. For use with async.add_timeout_to_deferred From 31c7c29d4304f7577a1f5dc1740fe3a248351500 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 3 May 2018 10:38:08 +0100 Subject: [PATCH 101/310] Fix up grammar --- synapse/storage/events.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index a9269707df..05cde96afc 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -556,9 +556,9 @@ class EventsStore(EventsWorkerStore): state_groups_map[ctx.state_group] = ctx.current_state_ids - # We need to map the event_ids to their state groups. First, lets - # check if the event is one we're persisting and then we can pull the - # state group from its context. + # We need to map the event_ids to their state groups. First, let's + # check if the event is one we're persisting, in which case we can + # pull the state group from its context. # Otherwise we need to pull the state group from the database. # Set of events we need to fetch groups for. (We know none of the old From a0501ac57ea597efaa32e66c57c2f87036ec8b32 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 3 May 2018 10:51:39 +0100 Subject: [PATCH 102/310] Warn of potential client incompatibility from #3161 --- CHANGES.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 317846d2a2..94b83027e2 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,18 @@ +Changes in synapse +=============================== + +Potentially breaking change: + +* Make Client-Server API return 403 for invalid token (PR #3161). + + This changes the Client-server spec to return a 403 error code instead of 401 + when the access token is unrecognised. This is the behaviour required by the + specification, but some clients may be relying on the old, incorrect + behaviour. + + Thanks to @NotAFile for fixing this. + + Changes in synapse v0.28.1 (2018-05-01) ======================================= From d72faf2fad9cd4a2f7a82bc35d7e74c36fc49e4f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 3 May 2018 10:56:42 +0100 Subject: [PATCH 103/310] Fix changes warning --- CHANGES.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 94b83027e2..9d40b2ac1e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,9 +3,9 @@ Changes in synapse Potentially breaking change: -* Make Client-Server API return 403 for invalid token (PR #3161). +* Make Client-Server API return 401 for invalid token (PR #3161). - This changes the Client-server spec to return a 403 error code instead of 401 + This changes the Client-server spec to return a 401 error code instead of 403 when the access token is unrecognised. This is the behaviour required by the specification, but some clients may be relying on the old, incorrect behaviour. From 2e7a94c36ba75a94114e4a9fb1aa6d4e3444e8b4 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 3 May 2018 12:31:47 +0100 Subject: [PATCH 104/310] Don't abortConnection() if the transport connection has already closed. --- synapse/http/endpoint.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/synapse/http/endpoint.py b/synapse/http/endpoint.py index d8923c9abb..564ae4c10d 100644 --- a/synapse/http/endpoint.py +++ b/synapse/http/endpoint.py @@ -113,10 +113,15 @@ class _WrappedConnection(object): if time.time() - self.last_request >= 2.5 * 60: self.abort() # Abort the underlying TLS connection. The abort() method calls - # loseConnection() on the underlying TLS connection which tries to + # loseConnection() on the TLS connection which tries to # shutdown the connection cleanly. We call abortConnection() - # since that will promptly close the underlying TCP connection. - self.transport.abortConnection() + # since that will promptly close the TLS connection. + # + # In Twisted >18.4; the TLS connection will be None if it has closed + # which will make abortConnection() throw. Check that the TLS connection + # is not None before trying to close it. + if self.transport.getHandle() is not None: + self.transport.abortConnection() def request(self, request): self.last_request = time.time() From a7fe62f0cb7bdb40e5c9b7f09f694e9c0913610c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 3 May 2018 11:16:36 +0100 Subject: [PATCH 105/310] Fix logcontext leaks in rate limiter --- synapse/util/ratelimitutils.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/synapse/util/ratelimitutils.py b/synapse/util/ratelimitutils.py index 18424f6c36..0ab63c3d7d 100644 --- a/synapse/util/ratelimitutils.py +++ b/synapse/util/ratelimitutils.py @@ -18,7 +18,10 @@ from twisted.internet import defer from synapse.api.errors import LimitExceededError from synapse.util.async import sleep -from synapse.util.logcontext import run_in_background +from synapse.util.logcontext import ( + run_in_background, make_deferred_yieldable, + PreserveLoggingContext, +) import collections import contextlib @@ -176,6 +179,9 @@ class _PerHostRatelimiter(object): return r def on_err(r): + # XXX: why is this necessary? this is called before we start + # processing the request so why would the request be in + # current_processing? self.current_processing.discard(request_id) return r @@ -187,7 +193,7 @@ class _PerHostRatelimiter(object): ret_defer.addCallbacks(on_start, on_err) ret_defer.addBoth(on_both) - return ret_defer + return make_deferred_yieldable(ret_defer) def _on_exit(self, request_id): logger.debug( @@ -197,7 +203,12 @@ class _PerHostRatelimiter(object): self.current_processing.discard(request_id) try: request_id, deferred = self.ready_request_queue.popitem() + + # XXX: why do we do the following? the on_start callback above will + # do it for us. self.current_processing.add(request_id) - deferred.callback(None) + + with PreserveLoggingContext(): + deferred.callback(None) except KeyError: pass From 95b6912045b5d23ae823d6733beb7f315b010528 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 3 May 2018 15:51:04 +0100 Subject: [PATCH 106/310] Fix metrics that have integer value labels --- synapse/metrics/metric.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/metrics/metric.py b/synapse/metrics/metric.py index fbba94e633..98d5ca7b6c 100644 --- a/synapse/metrics/metric.py +++ b/synapse/metrics/metric.py @@ -324,4 +324,4 @@ def _escape_character(m): def _escape_label_value(value): """Takes a label value and escapes quotes, newlines and backslashes """ - return re.sub(r"([\n\"\\])", _escape_character, value) + return re.sub(r"([\n\"\\])", _escape_character, str(value)) From 6d8ec3462d33daa5d390fca206ec2e281c27c38f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 3 May 2018 16:25:05 +0100 Subject: [PATCH 107/310] Note that label values can be anything --- synapse/metrics/metric.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/metrics/metric.py b/synapse/metrics/metric.py index 98d5ca7b6c..f421e7a93f 100644 --- a/synapse/metrics/metric.py +++ b/synapse/metrics/metric.py @@ -71,7 +71,8 @@ class BaseMetric(object): """Render this metric for a single set of labels Args: - label_values (list[str]): values for each of the labels + label_values (list[object]): values for each of the labels, + (which get stringified). value: value of the metric at with these labels Returns: From 5addeaa02c432241af282900b19172ba7158118c Mon Sep 17 00:00:00 2001 From: kaiyou Date: Fri, 4 May 2018 21:23:01 +0200 Subject: [PATCH 108/310] Add Docker packaging in the author list --- AUTHORS.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/AUTHORS.rst b/AUTHORS.rst index 3dcb1c2a89..e13ac5ad34 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -60,3 +60,6 @@ Niklas Riekenbrauck Christoph Witzany * Add LDAP support for authentication + +Pierre Jaury +* Docker packaging \ No newline at end of file From 88868b283994b9a73634d7153fd7104257475349 Mon Sep 17 00:00:00 2001 From: Konstantinos Sideris Date: Sat, 5 May 2018 12:55:02 +0300 Subject: [PATCH 109/310] notifications: Convert next_token to string according to the spec Currently the parameter is serialized as an integer. Signed-off-by: Konstantinos Sideris --- synapse/rest/client/v2_alpha/notifications.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/notifications.py b/synapse/rest/client/v2_alpha/notifications.py index ec170109fe..66583d6778 100644 --- a/synapse/rest/client/v2_alpha/notifications.py +++ b/synapse/rest/client/v2_alpha/notifications.py @@ -88,7 +88,7 @@ class NotificationsServlet(RestServlet): pa["topological_ordering"], pa["stream_ordering"] ) returned_push_actions.append(returned_pa) - next_token = pa["stream_ordering"] + next_token = str(pa["stream_ordering"]) defer.returnValue((200, { "notifications": returned_push_actions, From bf98fa0864fe62dbf1303acc778912d69fd4a738 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 8 May 2018 15:58:35 +0100 Subject: [PATCH 110/310] Part user from rooms on account deactivate This implements this very crudely: this probably isn't viable because parting a user from all their rooms could take a long time, and if the HS gets restarted in that time the process will be aborted. --- synapse/handlers/deactivate_account.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index b1d3814909..387620c9c6 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2017 New Vector Ltd +# Copyright 2017, 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ from twisted.internet import defer from ._base import BaseHandler +from synapse.types import UserID, create_requester import logging @@ -27,6 +28,7 @@ class DeactivateAccountHandler(BaseHandler): super(DeactivateAccountHandler, self).__init__(hs) self._auth_handler = hs.get_auth_handler() self._device_handler = hs.get_device_handler() + self._room_member_handler = hs.get_room_member_handler() @defer.inlineCallbacks def deactivate_account(self, user_id): @@ -50,3 +52,15 @@ class DeactivateAccountHandler(BaseHandler): yield self.store.user_delete_threepids(user_id) yield self.store.user_set_password_hash(user_id, None) + + user = UserID.from_string(user_id) + + rooms_for_user = yield self.store.get_rooms_for_user(user_id) + for room_id in rooms_for_user: + yield self._room_member_handler.update_membership( + create_requester(user), + user, + room_id, + "leave", + ratelimit=False, + ) From 06c0d0ed081b56716135c4881e600861b2e8cad5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 8 May 2018 15:45:38 +0100 Subject: [PATCH 111/310] Split paginate_room_events storage function --- synapse/storage/stream.py | 100 +++++++++++++++++++++++++++----------- 1 file changed, 72 insertions(+), 28 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index f0784ba137..b57a8a7ef6 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -738,17 +738,28 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): def has_room_changed_since(self, room_id, stream_id): return self._events_stream_cache.has_entity_changed(room_id, stream_id) + def paginate_room_events_txn(self, txn, room_id, from_key, to_key=None, + direction='b', limit=-1, event_filter=None): + """Returns list of events before or after a given token. -class StreamStore(StreamWorkerStore): - def get_room_max_stream_ordering(self): - return self._stream_id_gen.get_current_token() + Args: + txn + room_id (str) + from_key (str): The token used to stream from + to_key (str|None): A token which if given limits the results to + only those before + direction(char): Either 'b' or 'f' to indicate whether we are + paginating forwards or backwards from `from_key`. + limit (int): The maximum number of events to return. Zero or less + means no limit. + event_filter (Filter|None): If provided filters the events to + those that match the filter. - def get_room_min_stream_ordering(self): - return self._backfill_id_gen.get_current_token() - - @defer.inlineCallbacks - def paginate_room_events(self, room_id, from_key, to_key=None, - direction='b', limit=-1, event_filter=None): + Returns: + tuple[list[dict], str]: Returns the results as a list of dicts and + a token that points to the end of the result set. The dicts have + the keys "event_id", "toplogical_ordering" and "stream_orderign". + """ # Tokens really represent positions between elements, but we use # the convention of pointing to the event before the gap. Hence # we have a bit of asymmetry when it comes to equalities. @@ -795,29 +806,54 @@ class StreamStore(StreamWorkerStore): "limit": limit_str } - def f(txn): - txn.execute(sql, args) + txn.execute(sql, args) - rows = self.cursor_to_dict(txn) + rows = self.cursor_to_dict(txn) - if rows: - topo = rows[-1]["topological_ordering"] - toke = rows[-1]["stream_ordering"] - if direction == 'b': - # Tokens are positions between events. - # This token points *after* the last event in the chunk. - # We need it to point to the event before it in the chunk - # when we are going backwards so we subtract one from the - # stream part. - toke -= 1 - next_token = str(RoomStreamToken(topo, toke)) - else: - # TODO (erikj): We should work out what to do here instead. - next_token = to_key if to_key else from_key + if rows: + topo = rows[-1]["topological_ordering"] + toke = rows[-1]["stream_ordering"] + if direction == 'b': + # Tokens are positions between events. + # This token points *after* the last event in the chunk. + # We need it to point to the event before it in the chunk + # when we are going backwards so we subtract one from the + # stream part. + toke -= 1 + next_token = str(RoomStreamToken(topo, toke)) + else: + # TODO (erikj): We should work out what to do here instead. + next_token = to_key if to_key else from_key - return rows, next_token, + return rows, next_token, - rows, token = yield self.runInteraction("paginate_room_events", f) + @defer.inlineCallbacks + def paginate_room_events(self, room_id, from_key, to_key=None, + direction='b', limit=-1, event_filter=None): + """Returns list of events before or after a given token. + + Args: + room_id (str) + from_key (str): The token used to stream from + to_key (str|None): A token which if given limits the results to + only those before + direction(char): Either 'b' or 'f' to indicate whether we are + paginating forwards or backwards from `from_key`. + limit (int): The maximum number of events to return. Zero or less + means no limit. + event_filter (Filter|None): If provided filters the events to + those that match the filter. + + Returns: + tuple[list[dict], str]: Returns the results as a list of dicts and + a token that points to the end of the result set. The dicts have + the keys "event_id", "toplogical_ordering" and "stream_orderign". + """ + + rows, token = yield self.runInteraction( + "paginate_room_events", self.paginate_room_events_txn, + room_id, from_key, to_key, direction, limit, event_filter, + ) events = yield self._get_events( [r["event_id"] for r in rows], @@ -827,3 +863,11 @@ class StreamStore(StreamWorkerStore): self._set_before_and_after(events, rows) defer.returnValue((events, token)) + + +class StreamStore(StreamWorkerStore): + def get_room_max_stream_ordering(self): + return self._stream_id_gen.get_current_token() + + def get_room_min_stream_ordering(self): + return self._backfill_id_gen.get_current_token() From 274b8c6025e15eede0137bcb6a73ac00bf370ee8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 8 May 2018 16:15:07 +0100 Subject: [PATCH 112/310] Only fetch required fields from database --- synapse/storage/stream.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index b57a8a7ef6..d3adb0bf37 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -796,7 +796,8 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): limit_str = "" sql = ( - "SELECT * FROM events" + "SELECT event_id, topological_ordering, stream_ordering" + " FROM events" " WHERE outlier = ? AND room_id = ? AND %(bounds)s" " ORDER BY topological_ordering %(order)s," " stream_ordering %(order)s %(limit)s" From 3e6d306e94327983b1f6b66cec9faace642d3f16 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 8 May 2018 16:18:58 +0100 Subject: [PATCH 113/310] Parse tokens before calling DB function --- synapse/storage/stream.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index d3adb0bf37..ce98587608 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -738,16 +738,16 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): def has_room_changed_since(self, room_id, stream_id): return self._events_stream_cache.has_entity_changed(room_id, stream_id) - def paginate_room_events_txn(self, txn, room_id, from_key, to_key=None, + def paginate_room_events_txn(self, txn, room_id, from_token, to_token=None, direction='b', limit=-1, event_filter=None): """Returns list of events before or after a given token. Args: txn room_id (str) - from_key (str): The token used to stream from - to_key (str|None): A token which if given limits the results to - only those before + from_token (RoomStreamToken): The token used to stream from + to_token (RoomStreamToken|None): A token which if given limits the + results to only those before direction(char): Either 'b' or 'f' to indicate whether we are paginating forwards or backwards from `from_key`. limit (int): The maximum number of events to return. Zero or less @@ -757,7 +757,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): Returns: tuple[list[dict], str]: Returns the results as a list of dicts and - a token that points to the end of the result set. The dicts have + a token that points to the end of the result set. The dicts haveq the keys "event_id", "toplogical_ordering" and "stream_orderign". """ # Tokens really represent positions between elements, but we use @@ -767,20 +767,20 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): if direction == 'b': order = "DESC" bounds = upper_bound( - RoomStreamToken.parse(from_key), self.database_engine + from_token, self.database_engine ) - if to_key: + if to_token: bounds = "%s AND %s" % (bounds, lower_bound( - RoomStreamToken.parse(to_key), self.database_engine + to_token, self.database_engine )) else: order = "ASC" bounds = lower_bound( - RoomStreamToken.parse(from_key), self.database_engine + from_token, self.database_engine ) - if to_key: + if to_token: bounds = "%s AND %s" % (bounds, upper_bound( - RoomStreamToken.parse(to_key), self.database_engine + to_token, self.database_engine )) filter_clause, filter_args = filter_to_clause(event_filter) @@ -821,12 +821,12 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): # when we are going backwards so we subtract one from the # stream part. toke -= 1 - next_token = str(RoomStreamToken(topo, toke)) + next_token = RoomStreamToken(topo, toke) else: # TODO (erikj): We should work out what to do here instead. - next_token = to_key if to_key else from_key + next_token = to_token if to_token else from_token - return rows, next_token, + return rows, str(next_token), @defer.inlineCallbacks def paginate_room_events(self, room_id, from_key, to_key=None, @@ -851,6 +851,10 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): the keys "event_id", "toplogical_ordering" and "stream_orderign". """ + from_key = RoomStreamToken.parse(from_key) + if to_key: + to_key = RoomStreamToken.parse(to_key) + rows, token = yield self.runInteraction( "paginate_room_events", self.paginate_room_events_txn, room_id, from_key, to_key, direction, limit, event_filter, From 696f5324539b014a5a9613c3a534abb2b3725dac Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 8 May 2018 16:19:33 +0100 Subject: [PATCH 114/310] Reuse existing pagination code for context API --- synapse/storage/stream.py | 90 +++++++-------------------------------- 1 file changed, 15 insertions(+), 75 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index ce98587608..938a8809dc 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -42,7 +42,7 @@ from synapse.util.caches.descriptors import cached from synapse.types import RoomStreamToken from synapse.util.caches.stream_change_cache import StreamChangeCache from synapse.util.logcontext import make_deferred_yieldable, run_in_background -from synapse.storage.engines import PostgresEngine, Sqlite3Engine +from synapse.storage.engines import PostgresEngine import abc import logging @@ -595,88 +595,28 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): retcols=["stream_ordering", "topological_ordering"], ) - token = RoomStreamToken( + # Paginating backwards includes the event at the token, but paginating + # forward doesn't. + before_token = RoomStreamToken( + results["topological_ordering"] - 1, + results["stream_ordering"], + ) + + after_token = RoomStreamToken( results["topological_ordering"], results["stream_ordering"], ) - if isinstance(self.database_engine, Sqlite3Engine): - # SQLite3 doesn't optimise ``(x < a) OR (x = a AND y < b)`` - # So we give pass it to SQLite3 as the UNION ALL of the two queries. - - query_before = ( - "SELECT topological_ordering, stream_ordering, event_id FROM events" - " WHERE room_id = ? AND topological_ordering < ?" - " UNION ALL" - " SELECT topological_ordering, stream_ordering, event_id FROM events" - " WHERE room_id = ? AND topological_ordering = ? AND stream_ordering < ?" - " ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ?" - ) - before_args = ( - room_id, token.topological, - room_id, token.topological, token.stream, - before_limit, - ) - - query_after = ( - "SELECT topological_ordering, stream_ordering, event_id FROM events" - " WHERE room_id = ? AND topological_ordering > ?" - " UNION ALL" - " SELECT topological_ordering, stream_ordering, event_id FROM events" - " WHERE room_id = ? AND topological_ordering = ? AND stream_ordering > ?" - " ORDER BY topological_ordering ASC, stream_ordering ASC LIMIT ?" - ) - after_args = ( - room_id, token.topological, - room_id, token.topological, token.stream, - after_limit, - ) - else: - query_before = ( - "SELECT topological_ordering, stream_ordering, event_id FROM events" - " WHERE room_id = ? AND %s" - " ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ?" - ) % (upper_bound(token, self.database_engine, inclusive=False),) - - before_args = (room_id, before_limit) - - query_after = ( - "SELECT topological_ordering, stream_ordering, event_id FROM events" - " WHERE room_id = ? AND %s" - " ORDER BY topological_ordering ASC, stream_ordering ASC LIMIT ?" - ) % (lower_bound(token, self.database_engine, inclusive=False),) - - after_args = (room_id, after_limit) - - txn.execute(query_before, before_args) - - rows = self.cursor_to_dict(txn) + rows, start_token = self.paginate_room_events_txn( + txn, room_id, before_token, direction='b', limit=before_limit, + ) events_before = [r["event_id"] for r in rows] - if rows: - start_token = str(RoomStreamToken( - rows[0]["topological_ordering"], - rows[0]["stream_ordering"] - 1, - )) - else: - start_token = str(RoomStreamToken( - token.topological, - token.stream - 1, - )) - - txn.execute(query_after, after_args) - - rows = self.cursor_to_dict(txn) + rows, end_token = self.paginate_room_events_txn( + txn, room_id, after_token, direction='f', limit=after_limit, + ) events_after = [r["event_id"] for r in rows] - if rows: - end_token = str(RoomStreamToken( - rows[-1]["topological_ordering"], - rows[-1]["stream_ordering"], - )) - else: - end_token = str(token) - return { "before": { "event_ids": events_before, From 6c957e26f06f5806ae2e69838865ee2990a75258 Mon Sep 17 00:00:00 2001 From: rubo77 Date: Wed, 9 May 2018 00:14:01 +0200 Subject: [PATCH 115/310] nuke-room-from-db.sh: added postgresql option and help --- scripts-dev/nuke-room-from-db.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts-dev/nuke-room-from-db.sh b/scripts-dev/nuke-room-from-db.sh index 1201d176c2..cd74d78d6a 100755 --- a/scripts-dev/nuke-room-from-db.sh +++ b/scripts-dev/nuke-room-from-db.sh @@ -6,9 +6,19 @@ ## Do not run it lightly. +set -e + +if [ "$1" == "-h" ] || [ "$1" == "" ]; then + echo "Call with ROOM_ID as first option and then pipe it into the database. So for instance you might run" + echo " nuke-room-from-db.sh | sqlite3 homeserver.db" + echo "or" + echo " nuke-room-from-db.sh | psql --dbname=synapse" + exit +fi + ROOMID="$1" -sqlite3 homeserver.db < Date: Wed, 9 May 2018 00:21:00 +0200 Subject: [PATCH 116/310] Dont nuke non-existing table event_search_content --- scripts-dev/nuke-room-from-db.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts-dev/nuke-room-from-db.sh b/scripts-dev/nuke-room-from-db.sh index 1201d176c2..53617f21f1 100755 --- a/scripts-dev/nuke-room-from-db.sh +++ b/scripts-dev/nuke-room-from-db.sh @@ -29,7 +29,6 @@ DELETE FROM state_groups WHERE room_id = '$ROOMID'; DELETE FROM state_groups_state WHERE room_id = '$ROOMID'; DELETE FROM receipts_graph WHERE room_id = '$ROOMID'; DELETE FROM receipts_linearized WHERE room_id = '$ROOMID'; -DELETE FROM event_search_content WHERE c1room_id = '$ROOMID'; DELETE FROM guest_access WHERE room_id = '$ROOMID'; DELETE FROM history_visibility WHERE room_id = '$ROOMID'; DELETE FROM room_tags WHERE room_id = '$ROOMID'; From d11b8b6b659957b28c2bbf8bc6ca060284097377 Mon Sep 17 00:00:00 2001 From: rubo77 Date: Wed, 9 May 2018 00:46:47 +0200 Subject: [PATCH 117/310] nuke-room-from-db.sh: nuke from table event_search too --- scripts-dev/nuke-room-from-db.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts-dev/nuke-room-from-db.sh b/scripts-dev/nuke-room-from-db.sh index 53617f21f1..2cfd71b0b8 100755 --- a/scripts-dev/nuke-room-from-db.sh +++ b/scripts-dev/nuke-room-from-db.sh @@ -29,6 +29,7 @@ DELETE FROM state_groups WHERE room_id = '$ROOMID'; DELETE FROM state_groups_state WHERE room_id = '$ROOMID'; DELETE FROM receipts_graph WHERE room_id = '$ROOMID'; DELETE FROM receipts_linearized WHERE room_id = '$ROOMID'; +DELETE FROM event_search WHERE room_id = '$ROOMID'; DELETE FROM guest_access WHERE room_id = '$ROOMID'; DELETE FROM history_visibility WHERE room_id = '$ROOMID'; DELETE FROM room_tags WHERE room_id = '$ROOMID'; From 23ec51c94ce90fd70931fbe98cadb7dd51a9db4e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 May 2018 09:55:19 +0100 Subject: [PATCH 118/310] Fix up comments and make function private --- synapse/storage/stream.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 938a8809dc..54be025401 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -607,12 +607,12 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): results["stream_ordering"], ) - rows, start_token = self.paginate_room_events_txn( + rows, start_token = self._paginate_room_events_txn( txn, room_id, before_token, direction='b', limit=before_limit, ) events_before = [r["event_id"] for r in rows] - rows, end_token = self.paginate_room_events_txn( + rows, end_token = self._paginate_room_events_txn( txn, room_id, after_token, direction='f', limit=after_limit, ) events_after = [r["event_id"] for r in rows] @@ -678,8 +678,8 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): def has_room_changed_since(self, room_id, stream_id): return self._events_stream_cache.has_entity_changed(room_id, stream_id) - def paginate_room_events_txn(self, txn, room_id, from_token, to_token=None, - direction='b', limit=-1, event_filter=None): + def _paginate_room_events_txn(self, txn, room_id, from_token, to_token=None, + direction='b', limit=-1, event_filter=None): """Returns list of events before or after a given token. Args: @@ -697,8 +697,8 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): Returns: tuple[list[dict], str]: Returns the results as a list of dicts and - a token that points to the end of the result set. The dicts haveq - the keys "event_id", "toplogical_ordering" and "stream_orderign". + a token that points to the end of the result set. The dicts have + the keys "event_id", "toplogical_ordering" and "stream_ordering". """ # Tokens really represent positions between elements, but we use # the convention of pointing to the event before the gap. Hence @@ -796,7 +796,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): to_key = RoomStreamToken.parse(to_key) rows, token = yield self.runInteraction( - "paginate_room_events", self.paginate_room_events_txn, + "paginate_room_events", self._paginate_room_events_txn, room_id, from_key, to_key, direction, limit, event_filter, ) From 27cf170558865d0afeea5d4c2524e8664c28323b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 May 2018 10:53:29 +0100 Subject: [PATCH 119/310] Refactor recent events func to use pagination func This also removes a cache that is unlikely to ever get hit. --- synapse/storage/stream.py | 81 +++++++++++++++------------------------ 1 file changed, 30 insertions(+), 51 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 54be025401..da43bb1321 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -38,7 +38,6 @@ from twisted.internet import defer from synapse.storage._base import SQLBaseStore from synapse.storage.events import EventsWorkerStore -from synapse.util.caches.descriptors import cached from synapse.types import RoomStreamToken from synapse.util.caches.stream_change_cache import StreamChangeCache from synapse.util.logcontext import make_deferred_yieldable, run_in_background @@ -363,61 +362,41 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): defer.returnValue((events, token)) - @cached(num_args=4) + @defer.inlineCallbacks def get_recent_event_ids_for_room(self, room_id, limit, end_token, from_token=None): + """Get the most recent events in the room in topological ordering. + + Args: + room_id (str) + limit (int) + end_token (str): The stream token representing now. + from_token(str|None): Token to not return events before, if given. + + Returns: + Deferred[tuple[list[dict], tuple[str, str]]]: Returns a list of + dicts (which include event_ids, etc), and a tuple for + `(start_token, end_token)` representing the range of rows + returned. + The returned events are in ascending order. + """ + # Allow a zero limit here, and no-op. + if limit == 0: + defer.returnValue(([], (end_token, end_token))) + end_token = RoomStreamToken.parse_stream_token(end_token) + if from_token is not None: + from_token = RoomStreamToken.parse(from_token) - if from_token is None: - sql = ( - "SELECT stream_ordering, topological_ordering, event_id" - " FROM events" - " WHERE room_id = ? AND stream_ordering <= ? AND outlier = ?" - " ORDER BY topological_ordering DESC, stream_ordering DESC" - " LIMIT ?" - ) - else: - from_token = RoomStreamToken.parse_stream_token(from_token) - sql = ( - "SELECT stream_ordering, topological_ordering, event_id" - " FROM events" - " WHERE room_id = ? AND stream_ordering > ?" - " AND stream_ordering <= ? AND outlier = ?" - " ORDER BY topological_ordering DESC, stream_ordering DESC" - " LIMIT ?" - ) - - def get_recent_events_for_room_txn(txn): - if from_token is None: - txn.execute(sql, (room_id, end_token.stream, False, limit,)) - else: - txn.execute(sql, ( - room_id, from_token.stream, end_token.stream, False, limit - )) - - rows = self.cursor_to_dict(txn) - - rows.reverse() # As we selected with reverse ordering - - if rows: - # Tokens are positions between events. - # This token points *after* the last event in the chunk. - # We need it to point to the event before it in the chunk - # since we are going backwards so we subtract one from the - # stream part. - topo = rows[0]["topological_ordering"] - toke = rows[0]["stream_ordering"] - 1 - start_token = str(RoomStreamToken(topo, toke)) - - token = (start_token, str(end_token)) - else: - token = (str(end_token), str(end_token)) - - return rows, token - - return self.runInteraction( - "get_recent_events_for_room", get_recent_events_for_room_txn + rows, token = yield self.runInteraction( + "get_recent_event_ids_for_room", self._paginate_room_events_txn, + room_id, from_token=end_token, to_token=from_token, limit=limit, ) + # We want to return the results in ascending order. + rows.reverse() + + defer.returnValue((rows, (token, str(end_token)))) + def get_room_event_after_stream_ordering(self, room_id, stream_ordering): """Gets details of the first event in a room at or after a stream ordering From 7dd13415dbf82edeaae0b94f835e20025964e1b4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 May 2018 10:58:16 +0100 Subject: [PATCH 120/310] Remove unused from_token param --- synapse/storage/stream.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index da43bb1321..ecd39074b8 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -346,9 +346,9 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): defer.returnValue(ret) @defer.inlineCallbacks - def get_recent_events_for_room(self, room_id, limit, end_token, from_token=None): + def get_recent_events_for_room(self, room_id, limit, end_token): rows, token = yield self.get_recent_event_ids_for_room( - room_id, limit, end_token, from_token + room_id, limit, end_token, ) logger.debug("stream before") @@ -363,14 +363,13 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): defer.returnValue((events, token)) @defer.inlineCallbacks - def get_recent_event_ids_for_room(self, room_id, limit, end_token, from_token=None): + def get_recent_event_ids_for_room(self, room_id, limit, end_token): """Get the most recent events in the room in topological ordering. Args: room_id (str) limit (int) end_token (str): The stream token representing now. - from_token(str|None): Token to not return events before, if given. Returns: Deferred[tuple[list[dict], tuple[str, str]]]: Returns a list of @@ -384,12 +383,10 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): defer.returnValue(([], (end_token, end_token))) end_token = RoomStreamToken.parse_stream_token(end_token) - if from_token is not None: - from_token = RoomStreamToken.parse(from_token) rows, token = yield self.runInteraction( "get_recent_event_ids_for_room", self._paginate_room_events_txn, - room_id, from_token=end_token, to_token=from_token, limit=limit, + room_id, from_token=end_token, limit=limit, ) # We want to return the results in ascending order. From 05e0a2462c76be6987c7ec3d9517d500583bac65 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 May 2018 11:18:23 +0100 Subject: [PATCH 121/310] Refactor pagination DB API to return concrete type This makes it easier to document what is being returned by the storage functions and what some functions expect as arguments. --- synapse/storage/stream.py | 76 ++++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 28 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index ecd39074b8..772d2c6198 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -47,6 +47,7 @@ import abc import logging from six.moves import range +from collections import namedtuple logger = logging.getLogger(__name__) @@ -59,6 +60,12 @@ _STREAM_TOKEN = "stream" _TOPOLOGICAL_TOKEN = "topological" +# Used as return values for pagination APIs +_EventDictReturn = namedtuple("_EventDictReturn", ( + "event_id", "topological_ordering", "stream_ordering", +)) + + def lower_bound(token, engine, inclusive=False): inclusive = "=" if inclusive else "" if token.topological is None: @@ -256,9 +263,13 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): " ORDER BY stream_ordering %s LIMIT ?" ) % (order,) txn.execute(sql, (room_id, from_id, to_id, limit)) + + rows = [_EventDictReturn(row[0], None, row[1]) for row in txn] else: sql = ( - "SELECT event_id, stream_ordering FROM events WHERE" + "SELECT event_id, topological_ordering, stream_ordering" + " FROM events" + " WHERE" " room_id = ?" " AND not outlier" " AND stream_ordering <= ?" @@ -266,14 +277,14 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): ) % (order, order,) txn.execute(sql, (room_id, to_id, limit)) - rows = self.cursor_to_dict(txn) + rows = [_EventDictReturn(row[0], row[1], row[2]) for row in txn] return rows rows = yield self.runInteraction("get_room_events_stream_for_room", f) ret = yield self._get_events( - [r["event_id"] for r in rows], + [r.event_id for r in rows], get_prev_content=True ) @@ -283,7 +294,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): ret.reverse() if rows: - key = "s%d" % min(r["stream_ordering"] for r in rows) + key = "s%d" % min(r.stream_ordering for r in rows) else: # Assume we didn't get anything because there was nothing to # get. @@ -330,14 +341,15 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): " ORDER BY stream_ordering ASC" ) txn.execute(sql, (user_id, to_id,)) - rows = self.cursor_to_dict(txn) + + rows = [_EventDictReturn(row[0], None, row[1]) for row in txn] return rows rows = yield self.runInteraction("get_membership_changes_for_user", f) ret = yield self._get_events( - [r["event_id"] for r in rows], + [r.event_id for r in rows], get_prev_content=True ) @@ -353,14 +365,14 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): logger.debug("stream before") events = yield self._get_events( - [r["event_id"] for r in rows], + [r.event_id for r in rows], get_prev_content=True ) logger.debug("stream after") self._set_before_and_after(events, rows) - defer.returnValue((events, token)) + defer.returnValue((events, (token, end_token))) @defer.inlineCallbacks def get_recent_event_ids_for_room(self, room_id, limit, end_token): @@ -372,15 +384,14 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): end_token (str): The stream token representing now. Returns: - Deferred[tuple[list[dict], tuple[str, str]]]: Returns a list of - dicts (which include event_ids, etc), and a tuple for - `(start_token, end_token)` representing the range of rows - returned. - The returned events are in ascending order. + Deferred[tuple[list[_EventDictReturn], str]]: Returns a list of + _EventDictReturn and a token pointint to the start of the returned + events. + The events returned are in ascending order. """ # Allow a zero limit here, and no-op. if limit == 0: - defer.returnValue(([], (end_token, end_token))) + defer.returnValue(([], end_token)) end_token = RoomStreamToken.parse_stream_token(end_token) @@ -392,7 +403,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): # We want to return the results in ascending order. rows.reverse() - defer.returnValue((rows, (token, str(end_token)))) + defer.returnValue((rows, token)) def get_room_event_after_stream_ordering(self, room_id, stream_ordering): """Gets details of the first event in a room at or after a stream ordering @@ -496,10 +507,19 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): @staticmethod def _set_before_and_after(events, rows, topo_order=True): + """Inserts ordering information to events' internal metadata from + the DB rows. + + Args: + events (list[FrozenEvent]) + rows (list[_EventDictReturn]) + topo_order (bool): Whether the events were ordered topologically + or by stream ordering + """ for event, row in zip(events, rows): - stream = row["stream_ordering"] - if topo_order: - topo = event.depth + stream = row.stream_ordering + if topo_order and row.topological_ordering: + topo = row.topological_ordering else: topo = None internal = event.internal_metadata @@ -586,12 +606,12 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): rows, start_token = self._paginate_room_events_txn( txn, room_id, before_token, direction='b', limit=before_limit, ) - events_before = [r["event_id"] for r in rows] + events_before = [r.event_id for r in rows] rows, end_token = self._paginate_room_events_txn( txn, room_id, after_token, direction='f', limit=after_limit, ) - events_after = [r["event_id"] for r in rows] + events_after = [r.event_id for r in rows] return { "before": { @@ -672,9 +692,9 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): those that match the filter. Returns: - tuple[list[dict], str]: Returns the results as a list of dicts and - a token that points to the end of the result set. The dicts have - the keys "event_id", "toplogical_ordering" and "stream_ordering". + tuple[list[_EventDictReturn], str]: Returns the results as a list + of _EventDictReturn and a token that points to the end of the + result set. """ # Tokens really represent positions between elements, but we use # the convention of pointing to the event before the gap. Hence @@ -725,11 +745,11 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): txn.execute(sql, args) - rows = self.cursor_to_dict(txn) + rows = [_EventDictReturn(row[0], row[1], row[2]) for row in txn] if rows: - topo = rows[-1]["topological_ordering"] - toke = rows[-1]["stream_ordering"] + topo = rows[-1].topological_ordering + toke = rows[-1].stream_ordering if direction == 'b': # Tokens are positions between events. # This token points *after* the last event in the chunk. @@ -764,7 +784,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): Returns: tuple[list[dict], str]: Returns the results as a list of dicts and a token that points to the end of the result set. The dicts have - the keys "event_id", "toplogical_ordering" and "stream_orderign". + the keys "event_id", "topological_ordering" and "stream_orderign". """ from_key = RoomStreamToken.parse(from_key) @@ -777,7 +797,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): ) events = yield self._get_events( - [r["event_id"] for r in rows], + [r.event_id for r in rows], get_prev_content=True ) From c4af4c24ca988832018feaf0ac5a2f6dbb8bfe68 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 May 2018 11:55:34 +0100 Subject: [PATCH 122/310] Refactor get_recent_events_for_room return type There is no reason to return a tuple of tokens when the last token is always the token passed as an argument. Changing it makes it consistent with other storage APIs --- synapse/handlers/initial_sync.py | 10 +++++----- synapse/handlers/sync.py | 2 +- synapse/storage/stream.py | 16 +++++++++++++++- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py index cd33a86599..5a9aa0c16d 100644 --- a/synapse/handlers/initial_sync.py +++ b/synapse/handlers/initial_sync.py @@ -181,8 +181,8 @@ class InitialSyncHandler(BaseHandler): self.store, user_id, messages ) - start_token = now_token.copy_and_replace("room_key", token[0]) - end_token = now_token.copy_and_replace("room_key", token[1]) + start_token = now_token.copy_and_replace("room_key", token) + end_token = now_token.copy_and_replace("room_key", room_end_token) time_now = self.clock.time_msec() d["messages"] = { @@ -325,8 +325,8 @@ class InitialSyncHandler(BaseHandler): self.store, user_id, messages, is_peeking=is_peeking ) - start_token = StreamToken.START.copy_and_replace("room_key", token[0]) - end_token = StreamToken.START.copy_and_replace("room_key", token[1]) + start_token = StreamToken.START.copy_and_replace("room_key", token) + end_token = StreamToken.START.copy_and_replace("room_key", stream_token) time_now = self.clock.time_msec() @@ -409,7 +409,7 @@ class InitialSyncHandler(BaseHandler): ) start_token = now_token.copy_and_replace("room_key", token[0]) - end_token = now_token.copy_and_replace("room_key", token[1]) + end_token = now_token time_now = self.clock.time_msec() diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index b52e4c2aff..c25a76d215 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -429,7 +429,7 @@ class SyncHandler(object): Returns: A Deferred map from ((type, state_key)->Event) """ - last_events, token = yield self.store.get_recent_events_for_room( + last_events, _ = yield self.store.get_recent_events_for_room( room_id, end_token=stream_position.room_key, limit=1, ) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 772d2c6198..b5baacd32c 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -359,6 +359,20 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): @defer.inlineCallbacks def get_recent_events_for_room(self, room_id, limit, end_token): + """Get the most recent events in the room in topological ordering. + + Args: + room_id (str) + limit (int) + end_token (str): The stream token representing now. + + Returns: + Deferred[tuple[list[FrozenEvent], str]]: Returns a list of + events and a token pointint to the start of the returned + events. + The events returned are in ascending order. + """ + rows, token = yield self.get_recent_event_ids_for_room( room_id, limit, end_token, ) @@ -372,7 +386,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): self._set_before_and_after(events, rows) - defer.returnValue((events, (token, end_token))) + defer.returnValue((events, token)) @defer.inlineCallbacks def get_recent_event_ids_for_room(self, room_id, limit, end_token): From e5ab9cd24b2f63c6ca00ae6354dbcbfcd9127dfb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 May 2018 11:58:35 +0100 Subject: [PATCH 123/310] Don't unnecessarily require token to be stream token This allows calling the `get_recent_event_ids_for_room` function in more situations. --- synapse/storage/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index b5baacd32c..5e4327bb96 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -407,7 +407,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): if limit == 0: defer.returnValue(([], end_token)) - end_token = RoomStreamToken.parse_stream_token(end_token) + end_token = RoomStreamToken.parse(end_token) rows, token = yield self.runInteraction( "get_recent_event_ids_for_room", self._paginate_room_events_txn, From e2accd7f1d21e34181dd4543eca30ad1ea971b4c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 May 2018 11:59:45 +0100 Subject: [PATCH 124/310] Refactor sync APIs to reuse pagination API The sync API often returns events in a topological rather than stream ordering, e.g. when the user joined the room or on initial sync. When this happens we can reuse existing pagination storage functions. --- synapse/handlers/sync.py | 19 ++++++---- synapse/storage/stream.py | 73 +++++++++++++++++++-------------------- 2 files changed, 48 insertions(+), 44 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index c25a76d215..b75daa340d 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -354,12 +354,19 @@ class SyncHandler(object): since_key = since_token.room_key while limited and len(recents) < timeline_limit and max_repeat: - events, end_key = yield self.store.get_room_events_stream_for_room( - room_id, - limit=load_limit + 1, - from_key=since_key, - to_key=end_key, - ) + if since_key: + events, end_key = yield self.store.get_room_events_stream_for_room( + room_id, + limit=load_limit + 1, + from_key=since_key, + to_key=end_key, + ) + else: + events, end_key = yield self.store.get_recent_events_for_room( + room_id, + limit=load_limit + 1, + end_token=end_key, + ) loaded_recents = sync_config.filter_collection.filter_room_timeline( events ) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 5e4327bb96..8bb4e85709 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -233,52 +233,49 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): @defer.inlineCallbacks def get_room_events_stream_for_room(self, room_id, from_key, to_key, limit=0, order='DESC'): - # Note: If from_key is None then we return in topological order. This - # is because in that case we're using this as a "get the last few messages - # in a room" function, rather than "get new messages since last sync" - if from_key is not None: - from_id = RoomStreamToken.parse_stream_token(from_key).stream - else: - from_id = None - to_id = RoomStreamToken.parse_stream_token(to_key).stream + """Get new room events in stream ordering since `from_key`. + + Args: + room_id (str) + from_key (str): Token from which no events are returned before + to_key (str): Token from which no events are returned after. (This + is typically the current stream token) + limit (int): Maximum number of events to return + order (str): Either "DESC" or "ASC". Determines which events are + returned when the result is limited. If "DESC" then the most + recent `limit` events are returned, otherwise returns the + oldest `limit` events. + + Returns: + Deferred[tuple[list[FrozenEvent], str]]: Returns the list of + events (in ascending order) and the token from the start of + the chunk of events returned. + """ if from_key == to_key: defer.returnValue(([], from_key)) - if from_id: - has_changed = yield self._events_stream_cache.has_entity_changed( - room_id, from_id - ) + from_id = RoomStreamToken.parse_stream_token(from_key).stream + to_id = RoomStreamToken.parse_stream_token(to_key).stream - if not has_changed: - defer.returnValue(([], from_key)) + has_changed = yield self._events_stream_cache.has_entity_changed( + room_id, from_id + ) + + if not has_changed: + defer.returnValue(([], from_key)) def f(txn): - if from_id is not None: - sql = ( - "SELECT event_id, stream_ordering FROM events WHERE" - " room_id = ?" - " AND not outlier" - " AND stream_ordering > ? AND stream_ordering <= ?" - " ORDER BY stream_ordering %s LIMIT ?" - ) % (order,) - txn.execute(sql, (room_id, from_id, to_id, limit)) - - rows = [_EventDictReturn(row[0], None, row[1]) for row in txn] - else: - sql = ( - "SELECT event_id, topological_ordering, stream_ordering" - " FROM events" - " WHERE" - " room_id = ?" - " AND not outlier" - " AND stream_ordering <= ?" - " ORDER BY topological_ordering %s, stream_ordering %s LIMIT ?" - ) % (order, order,) - txn.execute(sql, (room_id, to_id, limit)) - - rows = [_EventDictReturn(row[0], row[1], row[2]) for row in txn] + sql = ( + "SELECT event_id, stream_ordering FROM events WHERE" + " room_id = ?" + " AND not outlier" + " AND stream_ordering > ? AND stream_ordering <= ?" + " ORDER BY stream_ordering %s LIMIT ?" + ) % (order,) + txn.execute(sql, (room_id, from_id, to_id, limit)) + rows = [_EventDictReturn(row[0], None, row[1]) for row in txn] return rows rows = yield self.runInteraction("get_room_events_stream_for_room", f) From c0e08dc45b8996e88ff3fd6c7584c26f5ec2e839 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 May 2018 14:31:32 +0100 Subject: [PATCH 125/310] Remove unused code path from member change DB func The function is never called without a from_key, so we can remove all the handling for that scenario. --- synapse/storage/stream.py | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 772d2c6198..25aef32551 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -304,10 +304,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): @defer.inlineCallbacks def get_membership_changes_for_user(self, user_id, from_key, to_key): - if from_key is not None: - from_id = RoomStreamToken.parse_stream_token(from_key).stream - else: - from_id = None + from_id = RoomStreamToken.parse_stream_token(from_key).stream to_id = RoomStreamToken.parse_stream_token(to_key).stream if from_key == to_key: @@ -321,26 +318,15 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): defer.returnValue([]) def f(txn): - if from_id is not None: - sql = ( - "SELECT m.event_id, stream_ordering FROM events AS e," - " room_memberships AS m" - " WHERE e.event_id = m.event_id" - " AND m.user_id = ?" - " AND e.stream_ordering > ? AND e.stream_ordering <= ?" - " ORDER BY e.stream_ordering ASC" - ) - txn.execute(sql, (user_id, from_id, to_id,)) - else: - sql = ( - "SELECT m.event_id, stream_ordering FROM events AS e," - " room_memberships AS m" - " WHERE e.event_id = m.event_id" - " AND m.user_id = ?" - " AND stream_ordering <= ?" - " ORDER BY stream_ordering ASC" - ) - txn.execute(sql, (user_id, to_id,)) + sql = ( + "SELECT m.event_id, stream_ordering FROM events AS e," + " room_memberships AS m" + " WHERE e.event_id = m.event_id" + " AND m.user_id = ?" + " AND e.stream_ordering > ? AND e.stream_ordering <= ?" + " ORDER BY e.stream_ordering ASC" + ) + txn.execute(sql, (user_id, from_id, to_id,)) rows = [_EventDictReturn(row[0], None, row[1]) for row in txn] From 7e8726b8fb297f255ed1fd78f8e3be1d7f21dcc7 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 9 May 2018 14:54:28 +0100 Subject: [PATCH 126/310] Part deactivated users in the background One room at a time so we don't take out the whole server with leave events, and restart at server restart. --- synapse/handlers/deactivate_account.py | 35 +++++++++++++++++++++++++- synapse/storage/registration.py | 27 ++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 387620c9c6..5bb6c11e8f 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -12,10 +12,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer +from twisted.internet import defer, reactor from ._base import BaseHandler from synapse.types import UserID, create_requester +from synapse.util.logcontext import run_in_background import logging @@ -30,6 +31,10 @@ class DeactivateAccountHandler(BaseHandler): self._device_handler = hs.get_device_handler() self._room_member_handler = hs.get_room_member_handler() + self._user_parter_running = False + + reactor.callWhenRunning(self.start_user_parting) + @defer.inlineCallbacks def deactivate_account(self, user_id): """Deactivate a user's account @@ -53,10 +58,38 @@ class DeactivateAccountHandler(BaseHandler): yield self.store.user_delete_threepids(user_id) yield self.store.user_set_password_hash(user_id, None) + yield self.store.add_user_pending_deactivation(user_id) + + self.start_user_parting() + + def start_user_parting(self): + if not self._user_parter_running: + run_in_background(self.user_parter_loop()) + + @defer.inlineCallbacks + def user_parter_loop(self): + self._user_parter_running = True + logger.info("Starting user parter") + try: + while True: + user_id = yield self.store.get_user_pending_deactivation() + if user_id is None: + break + logger.info("User parter parting %r", user_id) + yield self.part_user(user_id) + yield self.store.del_user_pending_deactivation(user_id) + logger.info("User parter finished parting %r", user_id) + logger.info("User parter finished: stopping") + finally: + self._user_parter_running = False + + @defer.inlineCallbacks + def part_user(self, user_id): user = UserID.from_string(user_id) rooms_for_user = yield self.store.get_rooms_for_user(user_id) for room_id in rooms_for_user: + logger.info("User parter parting %r from %r", user_id, room_id) yield self._room_member_handler.update_membership( create_requester(user), user, diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index a50717db2d..de068a55d2 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -526,3 +526,30 @@ class RegistrationStore(RegistrationWorkerStore, except self.database_engine.module.IntegrityError: ret = yield self.get_3pid_guest_access_token(medium, address) defer.returnValue(ret) + + def add_user_pending_deactivation(self, user_id): + return self._simple_insert( + "users_pending_deactivation", + values={ + "user_id": user_id, + }, + desc="add_user_pending_deactivation", + ) + + def del_user_pending_deactivation(self, user_id): + return self._simple_delete_one( + "users_pending_deactivation", + keyvalues={ + "user_id": user_id, + }, + desc="del_user_pending_deactivation", + ) + + def get_user_pending_deactivation(self): + return self._simple_select_one_onecol( + "users_pending_deactivation", + keyvalues={}, + retcol="user_id", + allow_none=True, + desc="get_users_pending_deactivation", + ) From 52281e4c548233087f03a014fdb21783a471b2a7 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 9 May 2018 15:06:16 +0100 Subject: [PATCH 127/310] Indent fail --- synapse/handlers/deactivate_account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 5bb6c11e8f..f99935b4ee 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -63,7 +63,7 @@ class DeactivateAccountHandler(BaseHandler): self.start_user_parting() def start_user_parting(self): - if not self._user_parter_running: + if not self._user_parter_running: run_in_background(self.user_parter_loop()) @defer.inlineCallbacks From 46df23f581bc52313cb1c9366e967ebcddef07cf Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 9 May 2018 15:07:54 +0100 Subject: [PATCH 128/310] Add the schema file --- .../schema/delta/48/deactivated_users.sql | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 synapse/storage/schema/delta/48/deactivated_users.sql diff --git a/synapse/storage/schema/delta/48/deactivated_users.sql b/synapse/storage/schema/delta/48/deactivated_users.sql new file mode 100644 index 0000000000..e9013a6969 --- /dev/null +++ b/synapse/storage/schema/delta/48/deactivated_users.sql @@ -0,0 +1,25 @@ +/* Copyright 2018 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Store any accounts that have been requested to be deactivated. + * We part the account from all the rooms its in when its + * deactivated. This can take some time and synapse may be restarted + * before it completes, so store the user IDs here until the process + * is complete. + */ +CREATE TABLE users_pending_deactivation ( + user_id TEXT NOT NULL +); From 294e9a0c9ba0c4ad69605eb781d9ae56b8982a96 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 9 May 2018 15:10:37 +0100 Subject: [PATCH 129/310] Prefix internal functions --- synapse/handlers/deactivate_account.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index f99935b4ee..81e0a75e78 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -33,7 +33,7 @@ class DeactivateAccountHandler(BaseHandler): self._user_parter_running = False - reactor.callWhenRunning(self.start_user_parting) + reactor.callWhenRunning(self._start_user_parting) @defer.inlineCallbacks def deactivate_account(self, user_id): @@ -60,14 +60,14 @@ class DeactivateAccountHandler(BaseHandler): yield self.store.add_user_pending_deactivation(user_id) - self.start_user_parting() + self._start_user_parting() - def start_user_parting(self): + def _start_user_parting(self): if not self._user_parter_running: - run_in_background(self.user_parter_loop()) + run_in_background(self._user_parter_loop()) @defer.inlineCallbacks - def user_parter_loop(self): + def _user_parter_loop(self): self._user_parter_running = True logger.info("Starting user parter") try: @@ -76,7 +76,7 @@ class DeactivateAccountHandler(BaseHandler): if user_id is None: break logger.info("User parter parting %r", user_id) - yield self.part_user(user_id) + yield self._part_user(user_id) yield self.store.del_user_pending_deactivation(user_id) logger.info("User parter finished parting %r", user_id) logger.info("User parter finished: stopping") @@ -84,7 +84,7 @@ class DeactivateAccountHandler(BaseHandler): self._user_parter_running = False @defer.inlineCallbacks - def part_user(self, user_id): + def _part_user(self, user_id): user = UserID.from_string(user_id) rooms_for_user = yield self.store.get_rooms_for_user(user_id) From 75552d2148f9f48add495cb7b18a378b0abf70b9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 May 2018 15:15:38 +0100 Subject: [PATCH 130/310] Update comments --- synapse/storage/stream.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 772d2c6198..60d2dca154 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -385,7 +385,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): Returns: Deferred[tuple[list[_EventDictReturn], str]]: Returns a list of - _EventDictReturn and a token pointint to the start of the returned + _EventDictReturn and a token pointing to the start of the returned events. The events returned are in ascending order. """ @@ -514,7 +514,8 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): events (list[FrozenEvent]) rows (list[_EventDictReturn]) topo_order (bool): Whether the events were ordered topologically - or by stream ordering + or by stream ordering. If true then all rows should have a non + null topological_ordering. """ for event, row in zip(events, rows): stream = row.stream_ordering @@ -692,9 +693,9 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): those that match the filter. Returns: - tuple[list[_EventDictReturn], str]: Returns the results as a list - of _EventDictReturn and a token that points to the end of the - result set. + Deferred[tuple[list[_EventDictReturn], str]]: Returns the results + as a list of _EventDictReturn and a token that points to the end + of the result set. """ # Tokens really represent positions between elements, but we use # the convention of pointing to the event before the gap. Hence From 7ce98804ff2ffa8ecacf63ddad469dc0221b655e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 May 2018 15:42:39 +0100 Subject: [PATCH 131/310] Fix up comment --- synapse/storage/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index f529642101..b6d171eb9b 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -368,7 +368,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): Returns: Deferred[tuple[list[FrozenEvent], str]]: Returns a list of - events and a token pointint to the start of the returned + events and a token pointing to the start of the returned events. The events returned are in ascending order. """ From fcf55f225541bcffa4bcb32213e0fde74f7b5827 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 May 2018 15:43:00 +0100 Subject: [PATCH 132/310] Fix returned token is no longer a tuple --- synapse/handlers/initial_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py index 5a9aa0c16d..71af86fe21 100644 --- a/synapse/handlers/initial_sync.py +++ b/synapse/handlers/initial_sync.py @@ -408,7 +408,7 @@ class InitialSyncHandler(BaseHandler): self.store, user_id, messages, is_peeking=is_peeking, ) - start_token = now_token.copy_and_replace("room_key", token[0]) + start_token = now_token.copy_and_replace("room_key", token) end_token = now_token time_now = self.clock.time_msec() From d26bec8a437e44a05febd02473a8dcd8ff2c9f58 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 May 2018 15:56:07 +0100 Subject: [PATCH 133/310] Add comment to sync as to why code path is split --- synapse/handlers/sync.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index b75daa340d..263e42dded 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -354,6 +354,11 @@ class SyncHandler(object): since_key = since_token.room_key while limited and len(recents) < timeline_limit and max_repeat: + # If we have a since_key then we are trying to get any events + # that have happened since `since_key` up to `end_key`, so we + # can just use `get_room_events_stream_for_room`. + # Otherwise, we want to return the last N events in the room + # in toplogical ordering. if since_key: events, end_key = yield self.store.get_room_events_stream_for_room( room_id, From 18e144fe088bda9e28697062cda62dca1e03724b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 9 May 2018 19:55:03 +0100 Subject: [PATCH 134/310] Move RequestsMetrics to its own file This is useful in its own right, because server.py is full of stuff; but more importantly, I want to do some refactoring that will cause a circular reference as it is. --- synapse/http/request_metrics.py | 147 ++++++++++++++++++++++++++++++++ synapse/http/server.py | 128 +-------------------------- 2 files changed, 151 insertions(+), 124 deletions(-) create mode 100644 synapse/http/request_metrics.py diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py new file mode 100644 index 0000000000..4a843a36a7 --- /dev/null +++ b/synapse/http/request_metrics.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- +# Copyright 2014-2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import synapse.metrics +from synapse.util.logcontext import LoggingContext + +logger = logging.getLogger(__name__) + +metrics = synapse.metrics.get_metrics_for("synapse.http.server") + +# total number of responses served, split by method/servlet/tag +response_count = metrics.register_counter( + "response_count", + labels=["method", "servlet", "tag"], + alternative_names=( + # the following are all deprecated aliases for the same metric + metrics.name_prefix + x for x in ( + "_requests", + "_response_time:count", + "_response_ru_utime:count", + "_response_ru_stime:count", + "_response_db_txn_count:count", + "_response_db_txn_duration:count", + ) + ) +) + +requests_counter = metrics.register_counter( + "requests_received", + labels=["method", "servlet", ], +) + +outgoing_responses_counter = metrics.register_counter( + "responses", + labels=["method", "code"], +) + +response_timer = metrics.register_counter( + "response_time_seconds", + labels=["method", "servlet", "tag"], + alternative_names=( + metrics.name_prefix + "_response_time:total", + ), +) + +response_ru_utime = metrics.register_counter( + "response_ru_utime_seconds", labels=["method", "servlet", "tag"], + alternative_names=( + metrics.name_prefix + "_response_ru_utime:total", + ), +) + +response_ru_stime = metrics.register_counter( + "response_ru_stime_seconds", labels=["method", "servlet", "tag"], + alternative_names=( + metrics.name_prefix + "_response_ru_stime:total", + ), +) + +response_db_txn_count = metrics.register_counter( + "response_db_txn_count", labels=["method", "servlet", "tag"], + alternative_names=( + metrics.name_prefix + "_response_db_txn_count:total", + ), +) + +# seconds spent waiting for db txns, excluding scheduling time, when processing +# this request +response_db_txn_duration = metrics.register_counter( + "response_db_txn_duration_seconds", labels=["method", "servlet", "tag"], + alternative_names=( + metrics.name_prefix + "_response_db_txn_duration:total", + ), +) + +# seconds spent waiting for a db connection, when processing this request +response_db_sched_duration = metrics.register_counter( + "response_db_sched_duration_seconds", labels=["method", "servlet", "tag"] +) + +# size in bytes of the response written +response_size = metrics.register_counter( + "response_size", labels=["method", "servlet", "tag"] +) + + +class RequestMetrics(object): + def start(self, clock, name): + self.start = clock.time_msec() + self.start_context = LoggingContext.current_context() + self.name = name + + def stop(self, clock, request): + context = LoggingContext.current_context() + + tag = "" + if context: + tag = context.tag + + if context != self.start_context: + logger.warn( + "Context have unexpectedly changed %r, %r", + context, self.start_context + ) + return + + response_count.inc(request.method, self.name, tag) + + response_timer.inc_by( + clock.time_msec() - self.start, request.method, + self.name, tag + ) + + ru_utime, ru_stime = context.get_resource_usage() + + response_ru_utime.inc_by( + ru_utime, request.method, self.name, tag + ) + response_ru_stime.inc_by( + ru_stime, request.method, self.name, tag + ) + response_db_txn_count.inc_by( + context.db_txn_count, request.method, self.name, tag + ) + response_db_txn_duration.inc_by( + context.db_txn_duration_ms / 1000., request.method, self.name, tag + ) + response_db_sched_duration.inc_by( + context.db_sched_duration_ms / 1000., request.method, self.name, tag + ) + + response_size.inc_by(request.sentLength, request.method, self.name, tag) diff --git a/synapse/http/server.py b/synapse/http/server.py index 55b9ad5251..37b26b908e 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -18,6 +18,10 @@ from synapse.api.errors import ( cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError, Codes ) +from synapse.http.request_metrics import ( + RequestMetrics, requests_counter, + outgoing_responses_counter, +) from synapse.util.logcontext import LoggingContext, PreserveLoggingContext from synapse.util.caches import intern_dict from synapse.util.metrics import Measure @@ -41,82 +45,6 @@ import simplejson logger = logging.getLogger(__name__) -metrics = synapse.metrics.get_metrics_for(__name__) - -# total number of responses served, split by method/servlet/tag -response_count = metrics.register_counter( - "response_count", - labels=["method", "servlet", "tag"], - alternative_names=( - # the following are all deprecated aliases for the same metric - metrics.name_prefix + x for x in ( - "_requests", - "_response_time:count", - "_response_ru_utime:count", - "_response_ru_stime:count", - "_response_db_txn_count:count", - "_response_db_txn_duration:count", - ) - ) -) - -requests_counter = metrics.register_counter( - "requests_received", - labels=["method", "servlet", ], -) - -outgoing_responses_counter = metrics.register_counter( - "responses", - labels=["method", "code"], -) - -response_timer = metrics.register_counter( - "response_time_seconds", - labels=["method", "servlet", "tag"], - alternative_names=( - metrics.name_prefix + "_response_time:total", - ), -) - -response_ru_utime = metrics.register_counter( - "response_ru_utime_seconds", labels=["method", "servlet", "tag"], - alternative_names=( - metrics.name_prefix + "_response_ru_utime:total", - ), -) - -response_ru_stime = metrics.register_counter( - "response_ru_stime_seconds", labels=["method", "servlet", "tag"], - alternative_names=( - metrics.name_prefix + "_response_ru_stime:total", - ), -) - -response_db_txn_count = metrics.register_counter( - "response_db_txn_count", labels=["method", "servlet", "tag"], - alternative_names=( - metrics.name_prefix + "_response_db_txn_count:total", - ), -) - -# seconds spent waiting for db txns, excluding scheduling time, when processing -# this request -response_db_txn_duration = metrics.register_counter( - "response_db_txn_duration_seconds", labels=["method", "servlet", "tag"], - alternative_names=( - metrics.name_prefix + "_response_db_txn_duration:total", - ), -) - -# seconds spent waiting for a db connection, when processing this request -response_db_sched_duration = metrics.register_counter( - "response_db_sched_duration_seconds", labels=["method", "servlet", "tag"] -) - -# size in bytes of the response written -response_size = metrics.register_counter( - "response_size", labels=["method", "servlet", "tag"] -) _next_request_id = 0 @@ -386,54 +314,6 @@ def _unrecognised_request_handler(request): raise UnrecognizedRequestError() -class RequestMetrics(object): - def start(self, clock, name): - self.start = clock.time_msec() - self.start_context = LoggingContext.current_context() - self.name = name - - def stop(self, clock, request): - context = LoggingContext.current_context() - - tag = "" - if context: - tag = context.tag - - if context != self.start_context: - logger.warn( - "Context have unexpectedly changed %r, %r", - context, self.start_context - ) - return - - response_count.inc(request.method, self.name, tag) - - response_timer.inc_by( - clock.time_msec() - self.start, request.method, - self.name, tag - ) - - ru_utime, ru_stime = context.get_resource_usage() - - response_ru_utime.inc_by( - ru_utime, request.method, self.name, tag - ) - response_ru_stime.inc_by( - ru_stime, request.method, self.name, tag - ) - response_db_txn_count.inc_by( - context.db_txn_count, request.method, self.name, tag - ) - response_db_txn_duration.inc_by( - context.db_txn_duration_ms / 1000., request.method, self.name, tag - ) - response_db_sched_duration.inc_by( - context.db_sched_duration_ms / 1000., request.method, self.name, tag - ) - - response_size.inc_by(request.sentLength, request.method, self.name, tag) - - class RootRedirect(resource.Resource): """Redirects the root '/' path to another path.""" From 8460e48d0639a7f575fd3e57dcd5cf263a6d6a19 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 9 May 2018 23:00:11 +0100 Subject: [PATCH 135/310] Move request_id management into SynapseRequest --- synapse/http/server.py | 31 ++++++++++++++++--------------- synapse/http/site.py | 9 +++++++++ 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index 37b26b908e..369f7bbdd1 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -46,38 +46,40 @@ import simplejson logger = logging.getLogger(__name__) -_next_request_id = 0 - - def request_handler(include_metrics=False): """Decorator for ``wrap_request_handler``""" return lambda request_handler: wrap_request_handler(request_handler, include_metrics) def wrap_request_handler(request_handler, include_metrics=False): - """Wraps a method that acts as a request handler with the necessary logging - and exception handling. + """Wraps a request handler method with the necessary logging and exception + handling. - The method must have a signature of "handle_foo(self, request)". The - argument "self" must have "version_string" and "clock" attributes. The - argument "request" must be a twisted HTTP request. + The handler method must have a signature of "handle_foo(self, request)", + where "self" must have "version_string" and "clock" attributes (and + "request" must be a SynapseRequest). - The method must return a deferred. If the deferred succeeds we assume that + The handler must return a deferred. If the deferred succeeds we assume that a response has been sent. If the deferred fails with a SynapseError we use it to send a JSON response with the appropriate HTTP reponse code. If the deferred fails with any other type of error we send a 500 reponse. - We insert a unique request-id into the logging context for this request and - log the response and duration for this request. + As well as calling `request.processing` (which will log the response and + duration for this request), the wrapped request handler will insert the + request id into the logging context. """ @defer.inlineCallbacks def wrapped_request_handler(self, request): - global _next_request_id - request_id = "%s-%s" % (request.method, _next_request_id) - _next_request_id += 1 + """ + Args: + self: + request (synapse.http.site.SynapseRequest): + """ + request_id = request.get_request_id() with LoggingContext(request_id) as request_context: + request_context.request = request_id with Measure(self.clock, "wrapped_request_handler"): request_metrics = RequestMetrics() # we start the request metrics timer here with an initial stab @@ -87,7 +89,6 @@ def wrap_request_handler(request_handler, include_metrics=False): servlet_name = self.__class__.__name__ request_metrics.start(self.clock, name=servlet_name) - request_context.request = request_id with request.processing(): try: with PreserveLoggingContext(request_context): diff --git a/synapse/http/site.py b/synapse/http/site.py index c8b46e1af2..6af276e69a 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -22,6 +22,8 @@ import time ACCESS_TOKEN_RE = re.compile(br'(\?.*access(_|%5[Ff])token=)[^&]*(.*)$') +_next_request_seq = 0 + class SynapseRequest(Request): def __init__(self, site, *args, **kw): @@ -30,6 +32,10 @@ class SynapseRequest(Request): self.authenticated_entity = None self.start_time = 0 + global _next_request_seq + self.request_seq = _next_request_seq + _next_request_seq += 1 + def __repr__(self): # We overwrite this so that we don't log ``access_token`` return '<%s at 0x%x method=%s uri=%s clientproto=%s site=%s>' % ( @@ -41,6 +47,9 @@ class SynapseRequest(Request): self.site.site_tag, ) + def get_request_id(self): + return "%s-%i" % (self.method, self.request_seq) + def get_redacted_uri(self): return ACCESS_TOKEN_RE.sub( br'\1\3', From 4d298506ddb65f642d899dffa236e4499f0c427b Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 10 May 2018 11:57:13 +0100 Subject: [PATCH 136/310] Oops, don't call function passed to run_in_background --- synapse/handlers/deactivate_account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 81e0a75e78..78558dd309 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -64,7 +64,7 @@ class DeactivateAccountHandler(BaseHandler): def _start_user_parting(self): if not self._user_parter_running: - run_in_background(self._user_parter_loop()) + run_in_background(self._user_parter_loop) @defer.inlineCallbacks def _user_parter_loop(self): From 09b29f9c4a941ae5721b1f5e296156dfe92e3395 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 9 May 2018 23:03:11 +0100 Subject: [PATCH 137/310] Make RequestMetrics take a raw time rather than a clock ... which is going to make it easier to move around. --- synapse/http/request_metrics.py | 8 ++++---- synapse/http/server.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py index 4a843a36a7..4e8a5f5306 100644 --- a/synapse/http/request_metrics.py +++ b/synapse/http/request_metrics.py @@ -100,12 +100,12 @@ response_size = metrics.register_counter( class RequestMetrics(object): - def start(self, clock, name): - self.start = clock.time_msec() + def start(self, time_msec, name): + self.start = time_msec self.start_context = LoggingContext.current_context() self.name = name - def stop(self, clock, request): + def stop(self, time_msec, request): context = LoggingContext.current_context() tag = "" @@ -122,7 +122,7 @@ class RequestMetrics(object): response_count.inc(request.method, self.name, tag) response_timer.inc_by( - clock.time_msec() - self.start, request.method, + time_msec - self.start, request.method, self.name, tag ) diff --git a/synapse/http/server.py b/synapse/http/server.py index 369f7bbdd1..b16c9c17f6 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -87,7 +87,7 @@ def wrap_request_handler(request_handler, include_metrics=False): # JsonResource (or a subclass), and JsonResource._async_render # will update it once it picks a servlet. servlet_name = self.__class__.__name__ - request_metrics.start(self.clock, name=servlet_name) + request_metrics.start(self.clock.time_msec(), name=servlet_name) with request.processing(): try: @@ -138,7 +138,7 @@ def wrap_request_handler(request_handler, include_metrics=False): finally: try: request_metrics.stop( - self.clock, request + self.clock.time_msec(), request ) except Exception as e: logger.warn("Failed to stop metrics: %r", e) From c6f730282c3a25468df0e624293aefd60cef7840 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 9 May 2018 23:05:14 +0100 Subject: [PATCH 138/310] Move RequestMetrics handling into SynapseRequest.processing() It fits quite nicely here, and opens the path to getting rid of the "include_metrics" mess. --- synapse/http/server.py | 19 ++++-------- synapse/http/site.py | 69 +++++++++++++++++++++++++++++++++++------- 2 files changed, 64 insertions(+), 24 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index b16c9c17f6..8e5d1d58f5 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -19,7 +19,7 @@ from synapse.api.errors import ( cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError, Codes ) from synapse.http.request_metrics import ( - RequestMetrics, requests_counter, + requests_counter, outgoing_responses_counter, ) from synapse.util.logcontext import LoggingContext, PreserveLoggingContext @@ -81,19 +81,18 @@ def wrap_request_handler(request_handler, include_metrics=False): with LoggingContext(request_id) as request_context: request_context.request = request_id with Measure(self.clock, "wrapped_request_handler"): - request_metrics = RequestMetrics() # we start the request metrics timer here with an initial stab # at the servlet name. For most requests that name will be # JsonResource (or a subclass), and JsonResource._async_render # will update it once it picks a servlet. servlet_name = self.__class__.__name__ - request_metrics.start(self.clock.time_msec(), name=servlet_name) - - with request.processing(): + with request.processing(servlet_name): try: with PreserveLoggingContext(request_context): if include_metrics: - yield request_handler(self, request, request_metrics) + yield request_handler( + self, request, request.request_metrics, + ) else: requests_counter.inc(request.method, servlet_name) yield request_handler(self, request) @@ -135,13 +134,7 @@ def wrap_request_handler(request_handler, include_metrics=False): pretty_print=_request_user_agent_is_curl(request), version_string=self.version_string, ) - finally: - try: - request_metrics.stop( - self.clock.time_msec(), request - ) - except Exception as e: - logger.warn("Failed to stop metrics: %r", e) + return wrapped_request_handler diff --git a/synapse/http/site.py b/synapse/http/site.py index 6af276e69a..bfd9832aa0 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -12,20 +12,38 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.util.logcontext import LoggingContext -from twisted.web.server import Site, Request - import contextlib import logging import re import time +from twisted.web.server import Site, Request + +from synapse.http.request_metrics import RequestMetrics +from synapse.util.logcontext import LoggingContext + +logger = logging.getLogger(__name__) + ACCESS_TOKEN_RE = re.compile(br'(\?.*access(_|%5[Ff])token=)[^&]*(.*)$') _next_request_seq = 0 class SynapseRequest(Request): + """Class which encapsulates an HTTP request to synapse. + + All of the requests processed in synapse are of this type. + + It extends twisted's twisted.web.server.Request, and adds: + * Unique request ID + * Redaction of access_token query-params in __repr__ + * Logging at start and end + * Metrics to record CPU, wallclock and DB time by endpoint. + + It provides a method `processing` which should be called by the Resource + which is handling the request, and returns a context manager. + + """ def __init__(self, site, *args, **kw): Request.__init__(self, *args, **kw) self.site = site @@ -59,7 +77,11 @@ class SynapseRequest(Request): def get_user_agent(self): return self.requestHeaders.getRawHeaders(b"User-Agent", [None])[-1] - def started_processing(self): + def _started_processing(self, servlet_name): + self.start_time = int(time.time() * 1000) + self.request_metrics = RequestMetrics() + self.request_metrics.start(self.start_time, name=servlet_name) + self.site.access_logger.info( "%s - %s - Received request: %s %s", self.getClientIP(), @@ -67,10 +89,8 @@ class SynapseRequest(Request): self.method, self.get_redacted_uri() ) - self.start_time = int(time.time() * 1000) - - def finished_processing(self): + def _finished_processing(self): try: context = LoggingContext.current_context() ru_utime, ru_stime = context.get_resource_usage() @@ -81,6 +101,8 @@ class SynapseRequest(Request): ru_utime, ru_stime = (0, 0) db_txn_count, db_txn_duration_ms = (0, 0) + end_time = int(time.time() * 1000) + self.site.access_logger.info( "%s - %s - {%s}" " Processed request: %dms (%dms, %dms) (%dms/%dms/%d)" @@ -88,7 +110,7 @@ class SynapseRequest(Request): self.getClientIP(), self.site.site_tag, self.authenticated_entity, - int(time.time() * 1000) - self.start_time, + end_time - self.start_time, int(ru_utime * 1000), int(ru_stime * 1000), db_sched_duration_ms, @@ -102,11 +124,36 @@ class SynapseRequest(Request): self.get_user_agent(), ) + try: + self.request_metrics.stop(end_time, self) + except Exception as e: + logger.warn("Failed to stop metrics: %r", e) + @contextlib.contextmanager - def processing(self): - self.started_processing() + def processing(self, servlet_name): + """Record the fact that we are processing this request. + + Returns a context manager; the correct way to use this is: + + @defer.inlineCallbacks + def handle_request(request): + with request.processing("FooServlet"): + yield really_handle_the_request() + + This will log the request's arrival. Once the context manager is + closed, the completion of the request will be logged, and the various + metrics will be updated. + + Args: + servlet_name (str): the name of the servlet which will be + processing this request. This is used in the metrics. + + It is possible to update this afterwards by updating + self.request_metrics.servlet_name. + """ + self._started_processing(servlet_name) yield - self.finished_processing() + self._finished_processing() class XForwardedForRequest(SynapseRequest): From b8700dd7d0482813beb9c780b411de5108ae078a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 9 May 2018 18:03:04 +0100 Subject: [PATCH 139/310] Bump requests_counter in wrapped_request_handler less magic --- synapse/http/server.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index 8e5d1d58f5..1d05b53873 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -90,12 +90,20 @@ def wrap_request_handler(request_handler, include_metrics=False): try: with PreserveLoggingContext(request_context): if include_metrics: - yield request_handler( + d = request_handler( self, request, request.request_metrics, ) else: - requests_counter.inc(request.method, servlet_name) - yield request_handler(self, request) + d = request_handler(self, request) + + # record the arrival of the request *after* + # dispatching to the handler, so that the handler + # can update the servlet name in the request + # metrics + requests_counter.inc(request.method, + request.request_metrics.name) + yield d + except CodeMessageException as e: code = e.code if isinstance(e, SynapseError): @@ -220,7 +228,6 @@ class JsonResource(HttpServer, resource.Resource): servlet_classname = "%r" % callback request_metrics.name = servlet_classname - requests_counter.inc(request.method, servlet_classname) # Now trigger the callback. If it returns a response, we send it # here. If it throws an exception, that is handled by the wrapper From 49e5a613f1ac53498b31150f78332fc64932e61b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 9 May 2018 23:44:22 +0100 Subject: [PATCH 140/310] Move outgoing_responses_counter handling to RequestMetrics it's much neater there. --- synapse/http/request_metrics.py | 2 ++ synapse/http/server.py | 4 ---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py index 4e8a5f5306..8c850bf23f 100644 --- a/synapse/http/request_metrics.py +++ b/synapse/http/request_metrics.py @@ -119,6 +119,8 @@ class RequestMetrics(object): ) return + outgoing_responses_counter.inc(request.method, str(request.code)) + response_count.inc(request.method, self.name, tag) response_timer.inc_by( diff --git a/synapse/http/server.py b/synapse/http/server.py index 1d05b53873..200b2c4837 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -20,7 +20,6 @@ from synapse.api.errors import ( ) from synapse.http.request_metrics import ( requests_counter, - outgoing_responses_counter, ) from synapse.util.logcontext import LoggingContext, PreserveLoggingContext from synapse.util.caches import intern_dict @@ -112,7 +111,6 @@ def wrap_request_handler(request_handler, include_metrics=False): ) else: logger.exception(e) - outgoing_responses_counter.inc(request.method, str(code)) respond_with_json( request, code, cs_exception(e), send_cors=True, pretty_print=_request_user_agent_is_curl(request), @@ -274,8 +272,6 @@ class JsonResource(HttpServer, resource.Resource): def _send_response(self, request, code, response_json_object, response_code_message=None): - outgoing_responses_counter.inc(request.method, str(code)) - # TODO: Only enable CORS for the requests that need it. respond_with_json( request, code, response_json_object, From 9589a1925e944360c68f7eb8a65f8ba94ec9bb84 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 9 May 2018 23:49:29 +0100 Subject: [PATCH 141/310] Remove include_metrics param The metrics are now available via the request, so this is redundant and can go away at last. --- synapse/http/server.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index 200b2c4837..9598969de8 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -45,12 +45,12 @@ import simplejson logger = logging.getLogger(__name__) -def request_handler(include_metrics=False): +def request_handler(): """Decorator for ``wrap_request_handler``""" - return lambda request_handler: wrap_request_handler(request_handler, include_metrics) + return wrap_request_handler -def wrap_request_handler(request_handler, include_metrics=False): +def wrap_request_handler(request_handler): """Wraps a request handler method with the necessary logging and exception handling. @@ -88,12 +88,7 @@ def wrap_request_handler(request_handler, include_metrics=False): with request.processing(servlet_name): try: with PreserveLoggingContext(request_context): - if include_metrics: - d = request_handler( - self, request, request.request_metrics, - ) - else: - d = request_handler(self, request) + d = request_handler(self, request) # record the arrival of the request *after* # dispatching to the handler, so that the handler @@ -206,13 +201,9 @@ class JsonResource(HttpServer, resource.Resource): self._async_render(request) return server.NOT_DONE_YET - # Disable metric reporting because _async_render does its own metrics. - # It does its own metric reporting because _async_render dispatches to - # a callback and it's the class name of that callback we want to report - # against rather than the JsonResource itself. - @request_handler(include_metrics=True) + @request_handler() @defer.inlineCallbacks - def _async_render(self, request, request_metrics): + def _async_render(self, request): """ This gets called from render() every time someone sends us a request. This checks if anyone has registered a callback for that method and path. @@ -224,8 +215,7 @@ class JsonResource(HttpServer, resource.Resource): servlet_classname = servlet_instance.__class__.__name__ else: servlet_classname = "%r" % callback - - request_metrics.name = servlet_classname + request.request_metrics.name = servlet_classname # Now trigger the callback. If it returns a response, we send it # here. If it throws an exception, that is handled by the wrapper From 09f570b9357d40764a0716f7e287fa4dde44610a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 10 May 2018 11:59:51 +0100 Subject: [PATCH 142/310] Factor wrap_request_handler_with_logging out of wrap_request_handler ... so that it can be used on non-JSON endpoints --- synapse/http/server.py | 118 +++++++++++++++++++++++------------------ 1 file changed, 65 insertions(+), 53 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index 9598969de8..fd58e65c4b 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -50,9 +50,10 @@ def request_handler(): return wrap_request_handler -def wrap_request_handler(request_handler): - """Wraps a request handler method with the necessary logging and exception - handling. +def wrap_request_handler(h): + """Wraps a request handler method with exception handling. + + Also adds logging as per wrap_request_handler_with_logging. The handler method must have a signature of "handle_foo(self, request)", where "self" must have "version_string" and "clock" attributes (and @@ -62,12 +63,63 @@ def wrap_request_handler(request_handler): a response has been sent. If the deferred fails with a SynapseError we use it to send a JSON response with the appropriate HTTP reponse code. If the deferred fails with any other type of error we send a 500 reponse. + """ + + @defer.inlineCallbacks + def wrapped_request_handler(self, request): + try: + yield h(self, request) + except CodeMessageException as e: + code = e.code + if isinstance(e, SynapseError): + logger.info( + "%s SynapseError: %s - %s", request, code, e.msg + ) + else: + logger.exception(e) + respond_with_json( + request, code, cs_exception(e), send_cors=True, + pretty_print=_request_user_agent_is_curl(request), + version_string=self.version_string, + ) + + except Exception: + # failure.Failure() fishes the original Failure out + # of our stack, and thus gives us a sensible stack + # trace. + f = failure.Failure() + logger.error( + "Failed handle request via %r: %r: %s", + h, + request, + f.getTraceback().rstrip(), + ) + respond_with_json( + request, + 500, + { + "error": "Internal server error", + "errcode": Codes.UNKNOWN, + }, + send_cors=True, + pretty_print=_request_user_agent_is_curl(request), + version_string=self.version_string, + ) + + return wrap_request_handler_with_logging(wrapped_request_handler) + + +def wrap_request_handler_with_logging(h): + """Wraps a request handler to provide logging and metrics + + The handler method must have a signature of "handle_foo(self, request)", + where "self" must have a "clock" attribute (and "request" must be a + SynapseRequest). As well as calling `request.processing` (which will log the response and duration for this request), the wrapped request handler will insert the request id into the logging context. """ - @defer.inlineCallbacks def wrapped_request_handler(self, request): """ @@ -86,56 +138,16 @@ def wrap_request_handler(request_handler): # will update it once it picks a servlet. servlet_name = self.__class__.__name__ with request.processing(servlet_name): - try: - with PreserveLoggingContext(request_context): - d = request_handler(self, request) - - # record the arrival of the request *after* - # dispatching to the handler, so that the handler - # can update the servlet name in the request - # metrics - requests_counter.inc(request.method, - request.request_metrics.name) - yield d - - except CodeMessageException as e: - code = e.code - if isinstance(e, SynapseError): - logger.info( - "%s SynapseError: %s - %s", request, code, e.msg - ) - else: - logger.exception(e) - respond_with_json( - request, code, cs_exception(e), send_cors=True, - pretty_print=_request_user_agent_is_curl(request), - version_string=self.version_string, - ) - except Exception: - # failure.Failure() fishes the original Failure out - # of our stack, and thus gives us a sensible stack - # trace. - f = failure.Failure() - logger.error( - "Failed handle request %s.%s on %r: %r: %s", - request_handler.__module__, - request_handler.__name__, - self, - request, - f.getTraceback().rstrip(), - ) - respond_with_json( - request, - 500, - { - "error": "Internal server error", - "errcode": Codes.UNKNOWN, - }, - send_cors=True, - pretty_print=_request_user_agent_is_curl(request), - version_string=self.version_string, - ) + with PreserveLoggingContext(request_context): + d = h(self, request) + # record the arrival of the request *after* + # dispatching to the handler, so that the handler + # can update the servlet name in the request + # metrics + requests_counter.inc(request.method, + request.request_metrics.name) + yield d return wrapped_request_handler From 645cb4bf06deee1c4c10ecc3d7df2c914168f19a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 10 May 2018 12:10:27 +0100 Subject: [PATCH 143/310] Remove redundant request_handler decorator This is needless complexity; we might as well use the wrapper directly. Also rename wrap_request_handler->wrap_json_request_handler. --- synapse/http/additional_resource.py | 4 ++-- synapse/http/server.py | 9 ++------ synapse/rest/key/v2/remote_key_resource.py | 8 ++++--- synapse/rest/media/v1/download_resource.py | 22 ++++++++++-------- synapse/rest/media/v1/preview_url_resource.py | 5 ++-- synapse/rest/media/v1/thumbnail_resource.py | 23 +++++++++++-------- synapse/rest/media/v1/upload_resource.py | 19 +++++++-------- 7 files changed, 47 insertions(+), 43 deletions(-) diff --git a/synapse/http/additional_resource.py b/synapse/http/additional_resource.py index 343e932cb1..d9e7f5dfb7 100644 --- a/synapse/http/additional_resource.py +++ b/synapse/http/additional_resource.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.http.server import wrap_request_handler +from synapse.http.server import wrap_json_request_handler from twisted.web.resource import Resource from twisted.web.server import NOT_DONE_YET @@ -50,6 +50,6 @@ class AdditionalResource(Resource): self._async_render(request) return NOT_DONE_YET - @wrap_request_handler + @wrap_json_request_handler def _async_render(self, request): return self._handler(request) diff --git a/synapse/http/server.py b/synapse/http/server.py index fd58e65c4b..f29e36f490 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -45,12 +45,7 @@ import simplejson logger = logging.getLogger(__name__) -def request_handler(): - """Decorator for ``wrap_request_handler``""" - return wrap_request_handler - - -def wrap_request_handler(h): +def wrap_json_request_handler(h): """Wraps a request handler method with exception handling. Also adds logging as per wrap_request_handler_with_logging. @@ -213,7 +208,7 @@ class JsonResource(HttpServer, resource.Resource): self._async_render(request) return server.NOT_DONE_YET - @request_handler() + @wrap_json_request_handler @defer.inlineCallbacks def _async_render(self, request): """ This gets called from render() every time someone sends us a request. diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py index 17e6079cba..17b3077926 100644 --- a/synapse/rest/key/v2/remote_key_resource.py +++ b/synapse/rest/key/v2/remote_key_resource.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.http.server import request_handler, respond_with_json_bytes +from synapse.http.server import ( + respond_with_json_bytes, wrap_json_request_handler, +) from synapse.http.servlet import parse_integer, parse_json_object_from_request from synapse.api.errors import SynapseError, Codes from synapse.crypto.keyring import KeyLookupError @@ -99,7 +101,7 @@ class RemoteKey(Resource): self.async_render_GET(request) return NOT_DONE_YET - @request_handler() + @wrap_json_request_handler @defer.inlineCallbacks def async_render_GET(self, request): if len(request.postpath) == 1: @@ -124,7 +126,7 @@ class RemoteKey(Resource): self.async_render_POST(request) return NOT_DONE_YET - @request_handler() + @wrap_json_request_handler @defer.inlineCallbacks def async_render_POST(self, request): content = parse_json_object_from_request(request) diff --git a/synapse/rest/media/v1/download_resource.py b/synapse/rest/media/v1/download_resource.py index fe7e17596f..3fc3f64d62 100644 --- a/synapse/rest/media/v1/download_resource.py +++ b/synapse/rest/media/v1/download_resource.py @@ -12,17 +12,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import synapse.http.servlet - -from ._base import parse_media_id, respond_404 -from twisted.web.resource import Resource -from synapse.http.server import request_handler, set_cors_headers - -from twisted.web.server import NOT_DONE_YET -from twisted.internet import defer - import logging +from twisted.internet import defer +from twisted.web.resource import Resource +from twisted.web.server import NOT_DONE_YET + +from synapse.http.server import ( + set_cors_headers, + wrap_json_request_handler, +) +import synapse.http.servlet +from ._base import parse_media_id, respond_404 + logger = logging.getLogger(__name__) @@ -43,7 +45,7 @@ class DownloadResource(Resource): self._async_render_GET(request) return NOT_DONE_YET - @request_handler() + @wrap_json_request_handler @defer.inlineCallbacks def _async_render_GET(self, request): set_cors_headers(request) diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py index 9290d7946f..6b089689b4 100644 --- a/synapse/rest/media/v1/preview_url_resource.py +++ b/synapse/rest/media/v1/preview_url_resource.py @@ -40,8 +40,9 @@ from synapse.util.stringutils import random_string from synapse.util.caches.expiringcache import ExpiringCache from synapse.http.client import SpiderHttpClient from synapse.http.server import ( - request_handler, respond_with_json_bytes, + respond_with_json_bytes, respond_with_json, + wrap_json_request_handler, ) from synapse.util.async import ObservableDeferred from synapse.util.stringutils import is_ascii @@ -90,7 +91,7 @@ class PreviewUrlResource(Resource): self._async_render_GET(request) return NOT_DONE_YET - @request_handler() + @wrap_json_request_handler @defer.inlineCallbacks def _async_render_GET(self, request): diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py index 58ada49711..6c12d79f56 100644 --- a/synapse/rest/media/v1/thumbnail_resource.py +++ b/synapse/rest/media/v1/thumbnail_resource.py @@ -14,18 +14,21 @@ # limitations under the License. +import logging + +from twisted.internet import defer +from twisted.web.resource import Resource +from twisted.web.server import NOT_DONE_YET + +from synapse.http.server import ( + set_cors_headers, + wrap_json_request_handler, +) +from synapse.http.servlet import parse_integer, parse_string from ._base import ( - parse_media_id, respond_404, respond_with_file, FileInfo, + FileInfo, parse_media_id, respond_404, respond_with_file, respond_with_responder, ) -from twisted.web.resource import Resource -from synapse.http.servlet import parse_string, parse_integer -from synapse.http.server import request_handler, set_cors_headers - -from twisted.web.server import NOT_DONE_YET -from twisted.internet import defer - -import logging logger = logging.getLogger(__name__) @@ -48,7 +51,7 @@ class ThumbnailResource(Resource): self._async_render_GET(request) return NOT_DONE_YET - @request_handler() + @wrap_json_request_handler @defer.inlineCallbacks def _async_render_GET(self, request): set_cors_headers(request) diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index a31e75cb46..7d01c57fd1 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -13,16 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.http.server import respond_with_json, request_handler +import logging + +from twisted.internet import defer +from twisted.web.resource import Resource +from twisted.web.server import NOT_DONE_YET from synapse.api.errors import SynapseError - -from twisted.web.server import NOT_DONE_YET -from twisted.internet import defer - -from twisted.web.resource import Resource - -import logging +from synapse.http.server import ( + respond_with_json, + wrap_json_request_handler, +) logger = logging.getLogger(__name__) @@ -51,7 +52,7 @@ class UploadResource(Resource): respond_with_json(request, 200, {}, send_cors=True) return NOT_DONE_YET - @request_handler() + @wrap_json_request_handler @defer.inlineCallbacks def _async_render_POST(self, request): requester = yield self.auth.get_user_by_req(request) From 217bc53c981c03ce880d1fa6afad2970be3f1d78 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 10 May 2018 12:20:40 +0100 Subject: [PATCH 144/310] Many docstrings --- synapse/handlers/deactivate_account.py | 24 ++++++++++++++++++++++++ synapse/storage/registration.py | 12 ++++++++++++ 2 files changed, 36 insertions(+) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 78558dd309..c00ec18627 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -31,8 +31,11 @@ class DeactivateAccountHandler(BaseHandler): self._device_handler = hs.get_device_handler() self._room_member_handler = hs.get_room_member_handler() + # Flag that indicates whether the process to part users from rooms is running self._user_parter_running = False + # Start the user parter loop so it can resume parting users from rooms where + # it left off (if it has work left to do). reactor.callWhenRunning(self._start_user_parting) @defer.inlineCallbacks @@ -58,16 +61,32 @@ class DeactivateAccountHandler(BaseHandler): yield self.store.user_delete_threepids(user_id) yield self.store.user_set_password_hash(user_id, None) + # Add the user to a table of users penpding deactivation (ie. + # removal from all the rooms they're a member of) yield self.store.add_user_pending_deactivation(user_id) + # Now start the process that goes through that list and + # parts users from rooms (if it isn't already running) self._start_user_parting() def _start_user_parting(self): + """ + Start the process that goes through the table of users + pending deactivation, if it isn't already running. + + Returns: + None + """ if not self._user_parter_running: run_in_background(self._user_parter_loop) @defer.inlineCallbacks def _user_parter_loop(self): + """Loop that parts deactivated users from rooms + + Returns: + None + """ self._user_parter_running = True logger.info("Starting user parter") try: @@ -85,6 +104,11 @@ class DeactivateAccountHandler(BaseHandler): @defer.inlineCallbacks def _part_user(self, user_id): + """Causes the given user_id to leave all the rooms they're joined to + + Returns: + None + """ user = UserID.from_string(user_id) rooms_for_user = yield self.store.get_rooms_for_user(user_id) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index de068a55d2..c05ce4612f 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -528,6 +528,10 @@ class RegistrationStore(RegistrationWorkerStore, defer.returnValue(ret) def add_user_pending_deactivation(self, user_id): + """ + Adds a user to the table of users who need to be parted from all the rooms they're + in + """ return self._simple_insert( "users_pending_deactivation", values={ @@ -537,6 +541,10 @@ class RegistrationStore(RegistrationWorkerStore, ) def del_user_pending_deactivation(self, user_id): + """ + Removes the given user to the table of users who need to be parted from all the + rooms they're in, effectively marking that user as fully deactivated. + """ return self._simple_delete_one( "users_pending_deactivation", keyvalues={ @@ -546,6 +554,10 @@ class RegistrationStore(RegistrationWorkerStore, ) def get_user_pending_deactivation(self): + """ + Gets one user from the table of users waiting to be parted from all the rooms + they're in. + """ return self._simple_select_one_onecol( "users_pending_deactivation", keyvalues={}, From 6b49628e3bf18f6cc1a1347fef8e4180e854d245 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 10 May 2018 12:23:53 +0100 Subject: [PATCH 145/310] Catch failure to part user from room --- synapse/handlers/deactivate_account.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index c00ec18627..4eb18775e8 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -114,10 +114,16 @@ class DeactivateAccountHandler(BaseHandler): rooms_for_user = yield self.store.get_rooms_for_user(user_id) for room_id in rooms_for_user: logger.info("User parter parting %r from %r", user_id, room_id) - yield self._room_member_handler.update_membership( - create_requester(user), - user, - room_id, - "leave", - ratelimit=False, - ) + try: + yield self._room_member_handler.update_membership( + create_requester(user), + user, + room_id, + "leave", + ratelimit=False, + ) + except Exception: + logger.exception( + "Failed to part user %r from room %r: ignoring and continuing", + user_id, room_id, + ) From 318711e1399da009910c3a9e5fa297c28a2d0a97 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 10 May 2018 18:46:59 +0100 Subject: [PATCH 146/310] Set Server header in SynapseRequest (instead of everywhere that writes a response. Or rather, the subset of places which write responses where we haven't forgotten it). This also means that we don't have to have the mysterious version_string attribute in anything with a request handler. Unfortunately it does mean that we have to pass the version string wherever we instantiate a SynapseSite, which has been c&ped 150 times, but that is code that ought to be cleaned up anyway really. --- synapse/app/appservice.py | 1 + synapse/app/client_reader.py | 1 + synapse/app/event_creator.py | 1 + synapse/app/federation_reader.py | 1 + synapse/app/federation_sender.py | 1 + synapse/app/frontend_proxy.py | 1 + synapse/app/homeserver.py | 2 ++ synapse/app/media_repository.py | 1 + synapse/app/pusher.py | 1 + synapse/app/synchrotron.py | 1 + synapse/app/user_dir.py | 1 + synapse/http/additional_resource.py | 3 +-- synapse/http/server.py | 14 ++++---------- synapse/http/site.py | 11 ++++++++++- synapse/rest/client/v1/pusher.py | 1 - synapse/rest/client/v2_alpha/auth.py | 2 -- synapse/rest/key/v1/server_key_resource.py | 2 -- synapse/rest/key/v2/local_key_resource.py | 2 -- synapse/rest/key/v2/remote_key_resource.py | 2 -- synapse/rest/media/v1/download_resource.py | 3 +-- synapse/rest/media/v1/preview_url_resource.py | 1 - synapse/rest/media/v1/thumbnail_resource.py | 1 - synapse/rest/media/v1/upload_resource.py | 1 - 23 files changed, 28 insertions(+), 27 deletions(-) diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py index 58f2c9d68c..b1efacc9f8 100644 --- a/synapse/app/appservice.py +++ b/synapse/app/appservice.py @@ -74,6 +74,7 @@ class AppserviceServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ) ) diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py index 267d34c881..38b98382c6 100644 --- a/synapse/app/client_reader.py +++ b/synapse/app/client_reader.py @@ -98,6 +98,7 @@ class ClientReaderServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ) ) diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py index b915d12d53..bd7f3d5679 100644 --- a/synapse/app/event_creator.py +++ b/synapse/app/event_creator.py @@ -114,6 +114,7 @@ class EventCreatorServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ) ) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index c1dc66dd17..6e10b27b9e 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -87,6 +87,7 @@ class FederationReaderServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ) ) diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py index a08af83a4c..6f24e32d6d 100644 --- a/synapse/app/federation_sender.py +++ b/synapse/app/federation_sender.py @@ -101,6 +101,7 @@ class FederationSenderServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ) ) diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py index b349e3e3ce..0f700ee786 100644 --- a/synapse/app/frontend_proxy.py +++ b/synapse/app/frontend_proxy.py @@ -152,6 +152,7 @@ class FrontendProxyServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ) ) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index a0e465d644..75f40fd5a4 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -140,6 +140,7 @@ class SynapseHomeServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ), self.tls_server_context_factory, ) @@ -153,6 +154,7 @@ class SynapseHomeServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ) ) logger.info("Synapse now listening on port %d", port) diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py index fc8282bbc1..9c93195f0a 100644 --- a/synapse/app/media_repository.py +++ b/synapse/app/media_repository.py @@ -94,6 +94,7 @@ class MediaRepositoryServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ) ) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index 26930d1b3b..3912eae48c 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -104,6 +104,7 @@ class PusherServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ) ) diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index 7152b1deb4..c6294a7a0c 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -281,6 +281,7 @@ class SynchrotronServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ) ) diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py index 5ba7e9b416..53eb3474da 100644 --- a/synapse/app/user_dir.py +++ b/synapse/app/user_dir.py @@ -126,6 +126,7 @@ class UserDirectoryServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ) ) diff --git a/synapse/http/additional_resource.py b/synapse/http/additional_resource.py index d9e7f5dfb7..a797396ade 100644 --- a/synapse/http/additional_resource.py +++ b/synapse/http/additional_resource.py @@ -42,8 +42,7 @@ class AdditionalResource(Resource): Resource.__init__(self) self._handler = handler - # these are required by the request_handler wrapper - self.version_string = hs.version_string + # required by the request_handler wrapper self.clock = hs.get_clock() def render(self, request): diff --git a/synapse/http/server.py b/synapse/http/server.py index f29e36f490..b6e2ae14a2 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -51,8 +51,8 @@ def wrap_json_request_handler(h): Also adds logging as per wrap_request_handler_with_logging. The handler method must have a signature of "handle_foo(self, request)", - where "self" must have "version_string" and "clock" attributes (and - "request" must be a SynapseRequest). + where "self" must have a "clock" attribute (and "request" must be a + SynapseRequest). The handler must return a deferred. If the deferred succeeds we assume that a response has been sent. If the deferred fails with a SynapseError we use @@ -75,7 +75,6 @@ def wrap_json_request_handler(h): respond_with_json( request, code, cs_exception(e), send_cors=True, pretty_print=_request_user_agent_is_curl(request), - version_string=self.version_string, ) except Exception: @@ -98,7 +97,6 @@ def wrap_json_request_handler(h): }, send_cors=True, pretty_print=_request_user_agent_is_curl(request), - version_string=self.version_string, ) return wrap_request_handler_with_logging(wrapped_request_handler) @@ -192,7 +190,6 @@ class JsonResource(HttpServer, resource.Resource): self.canonical_json = canonical_json self.clock = hs.get_clock() self.path_regexs = {} - self.version_string = hs.version_string self.hs = hs def register_paths(self, method, path_patterns, callback): @@ -275,7 +272,6 @@ class JsonResource(HttpServer, resource.Resource): send_cors=True, response_code_message=response_code_message, pretty_print=_request_user_agent_is_curl(request), - version_string=self.version_string, canonical_json=self.canonical_json, ) @@ -326,7 +322,7 @@ class RootRedirect(resource.Resource): def respond_with_json(request, code, json_object, send_cors=False, response_code_message=None, pretty_print=False, - version_string="", canonical_json=True): + canonical_json=True): # could alternatively use request.notifyFinish() and flip a flag when # the Deferred fires, but since the flag is RIGHT THERE it seems like # a waste. @@ -348,12 +344,11 @@ def respond_with_json(request, code, json_object, send_cors=False, request, code, json_bytes, send_cors=send_cors, response_code_message=response_code_message, - version_string=version_string ) def respond_with_json_bytes(request, code, json_bytes, send_cors=False, - version_string="", response_code_message=None): + response_code_message=None): """Sends encoded JSON in response to the given request. Args: @@ -367,7 +362,6 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False, request.setResponseCode(code, message=response_code_message) request.setHeader(b"Content-Type", b"application/json") - request.setHeader(b"Server", version_string) request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),)) request.setHeader(b"Cache-Control", b"no-cache, no-store, must-revalidate") diff --git a/synapse/http/site.py b/synapse/http/site.py index bfd9832aa0..202a990508 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -77,6 +77,11 @@ class SynapseRequest(Request): def get_user_agent(self): return self.requestHeaders.getRawHeaders(b"User-Agent", [None])[-1] + def render(self, resrc): + # override the Server header which is set by twisted + self.setHeader("Server", self.site.server_version_string) + return Request.render(self, resrc) + def _started_processing(self, servlet_name): self.start_time = int(time.time() * 1000) self.request_metrics = RequestMetrics() @@ -151,6 +156,8 @@ class SynapseRequest(Request): It is possible to update this afterwards by updating self.request_metrics.servlet_name. """ + # TODO: we should probably just move this into render() and finish(), + # to save having to call a separate method. self._started_processing(servlet_name) yield self._finished_processing() @@ -191,7 +198,8 @@ class SynapseSite(Site): Subclass of a twisted http Site that does access logging with python's standard logging """ - def __init__(self, logger_name, site_tag, config, resource, *args, **kwargs): + def __init__(self, logger_name, site_tag, config, resource, + server_version_string, *args, **kwargs): Site.__init__(self, resource, *args, **kwargs) self.site_tag = site_tag @@ -199,6 +207,7 @@ class SynapseSite(Site): proxied = config.get("x_forwarded", False) self.requestFactory = SynapseRequestFactory(self, proxied) self.access_logger = logging.getLogger(logger_name) + self.server_version_string = server_version_string def log(self, request): pass diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 0206e664c1..40e523cc5f 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -176,7 +176,6 @@ class PushersRemoveRestServlet(RestServlet): request.setResponseCode(200) request.setHeader(b"Content-Type", b"text/html; charset=utf-8") - request.setHeader(b"Server", self.hs.version_string) request.setHeader(b"Content-Length", b"%d" % ( len(PushersRemoveRestServlet.SUCCESS_HTML), )) diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/v2_alpha/auth.py index 8e5577148f..d6f3a19648 100644 --- a/synapse/rest/client/v2_alpha/auth.py +++ b/synapse/rest/client/v2_alpha/auth.py @@ -129,7 +129,6 @@ class AuthRestServlet(RestServlet): html_bytes = html.encode("utf8") request.setResponseCode(200) request.setHeader(b"Content-Type", b"text/html; charset=utf-8") - request.setHeader(b"Server", self.hs.version_string) request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),)) request.write(html_bytes) @@ -175,7 +174,6 @@ class AuthRestServlet(RestServlet): html_bytes = html.encode("utf8") request.setResponseCode(200) request.setHeader(b"Content-Type", b"text/html; charset=utf-8") - request.setHeader(b"Server", self.hs.version_string) request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),)) request.write(html_bytes) diff --git a/synapse/rest/key/v1/server_key_resource.py b/synapse/rest/key/v1/server_key_resource.py index bd4fea5774..1498d188c1 100644 --- a/synapse/rest/key/v1/server_key_resource.py +++ b/synapse/rest/key/v1/server_key_resource.py @@ -49,7 +49,6 @@ class LocalKey(Resource): """ def __init__(self, hs): - self.version_string = hs.version_string self.response_body = encode_canonical_json( self.response_json_object(hs.config) ) @@ -84,7 +83,6 @@ class LocalKey(Resource): def render_GET(self, request): return respond_with_json_bytes( request, 200, self.response_body, - version_string=self.version_string ) def getChild(self, name, request): diff --git a/synapse/rest/key/v2/local_key_resource.py b/synapse/rest/key/v2/local_key_resource.py index be68d9a096..04775b3c45 100644 --- a/synapse/rest/key/v2/local_key_resource.py +++ b/synapse/rest/key/v2/local_key_resource.py @@ -63,7 +63,6 @@ class LocalKey(Resource): isLeaf = True def __init__(self, hs): - self.version_string = hs.version_string self.config = hs.config self.clock = hs.clock self.update_response_body(self.clock.time_msec()) @@ -115,5 +114,4 @@ class LocalKey(Resource): self.update_response_body(time_now) return respond_with_json_bytes( request, 200, self.response_body, - version_string=self.version_string ) diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py index 17b3077926..21b4c1175e 100644 --- a/synapse/rest/key/v2/remote_key_resource.py +++ b/synapse/rest/key/v2/remote_key_resource.py @@ -93,7 +93,6 @@ class RemoteKey(Resource): def __init__(self, hs): self.keyring = hs.get_keyring() self.store = hs.get_datastore() - self.version_string = hs.version_string self.clock = hs.get_clock() self.federation_domain_whitelist = hs.config.federation_domain_whitelist @@ -242,5 +241,4 @@ class RemoteKey(Resource): respond_with_json_bytes( request, 200, result_io.getvalue(), - version_string=self.version_string ) diff --git a/synapse/rest/media/v1/download_resource.py b/synapse/rest/media/v1/download_resource.py index 3fc3f64d62..8cf8820c31 100644 --- a/synapse/rest/media/v1/download_resource.py +++ b/synapse/rest/media/v1/download_resource.py @@ -37,9 +37,8 @@ class DownloadResource(Resource): self.media_repo = media_repo self.server_name = hs.hostname - # Both of these are expected by @request_handler() + # this is expected by @wrap_json_request_handler self.clock = hs.get_clock() - self.version_string = hs.version_string def render_GET(self, request): self._async_render_GET(request) diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py index 6b089689b4..2839207abc 100644 --- a/synapse/rest/media/v1/preview_url_resource.py +++ b/synapse/rest/media/v1/preview_url_resource.py @@ -58,7 +58,6 @@ class PreviewUrlResource(Resource): self.auth = hs.get_auth() self.clock = hs.get_clock() - self.version_string = hs.version_string self.filepaths = media_repo.filepaths self.max_spider_size = hs.config.max_spider_size self.server_name = hs.hostname diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py index 6c12d79f56..aae6e464e8 100644 --- a/synapse/rest/media/v1/thumbnail_resource.py +++ b/synapse/rest/media/v1/thumbnail_resource.py @@ -44,7 +44,6 @@ class ThumbnailResource(Resource): self.media_storage = media_storage self.dynamic_thumbnails = hs.config.dynamic_thumbnails self.server_name = hs.hostname - self.version_string = hs.version_string self.clock = hs.get_clock() def render_GET(self, request): diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index 7d01c57fd1..7567476fce 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -41,7 +41,6 @@ class UploadResource(Resource): self.server_name = hs.hostname self.auth = hs.get_auth() self.max_upload_size = hs.config.max_upload_size - self.version_string = hs.version_string self.clock = hs.get_clock() def render_POST(self, request): From db18d854cd8e197cb3d77441b2a7ebd6f81beb81 Mon Sep 17 00:00:00 2001 From: Damir Manapov Date: Thu, 10 May 2018 22:13:31 +0300 Subject: [PATCH 147/310] transaction_id, destination twice --- synapse/federation/units.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/federation/units.py b/synapse/federation/units.py index 3f645acc43..01c5b8fe17 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -74,8 +74,6 @@ class Transaction(JsonEncodedObject): "previous_ids", "pdus", "edus", - "transaction_id", - "destination", "pdu_failures", ] From 6406b70aebd61ecddf61e45196d7cb13428b509e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 May 2018 15:30:11 +0100 Subject: [PATCH 148/310] Use stream rather depth ordering for push actions This simplifies things as it is, but will also allow us to change the way we traverse topologically without having to update the way push actions work. --- synapse/storage/event_push_actions.py | 53 +++++++----------------- synapse/storage/receipts.py | 1 - tests/storage/test_event_push_actions.py | 4 +- 3 files changed, 18 insertions(+), 40 deletions(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index c22762eb5c..f084a5f54b 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -18,8 +18,6 @@ from synapse.storage._base import SQLBaseStore, LoggingTransaction from twisted.internet import defer from synapse.util.async import sleep from synapse.util.caches.descriptors import cachedInlineCallbacks -from synapse.types import RoomStreamToken -from .stream import lower_bound import logging import simplejson as json @@ -99,7 +97,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): def _get_unread_counts_by_receipt_txn(self, txn, room_id, user_id, last_read_event_id): sql = ( - "SELECT stream_ordering, topological_ordering" + "SELECT stream_ordering" " FROM events" " WHERE room_id = ? AND event_id = ?" ) @@ -111,17 +109,12 @@ class EventPushActionsWorkerStore(SQLBaseStore): return {"notify_count": 0, "highlight_count": 0} stream_ordering = results[0][0] - topological_ordering = results[0][1] return self._get_unread_counts_by_pos_txn( - txn, room_id, user_id, topological_ordering, stream_ordering + txn, room_id, user_id, stream_ordering ) - def _get_unread_counts_by_pos_txn(self, txn, room_id, user_id, topological_ordering, - stream_ordering): - token = RoomStreamToken( - topological_ordering, stream_ordering - ) + def _get_unread_counts_by_pos_txn(self, txn, room_id, user_id, stream_ordering): # First get number of notifications. # We don't need to put a notif=1 clause as all rows always have @@ -132,10 +125,10 @@ class EventPushActionsWorkerStore(SQLBaseStore): " WHERE" " user_id = ?" " AND room_id = ?" - " AND %s" - ) % (lower_bound(token, self.database_engine, inclusive=False),) + " AND stream_ordering > ?" + ) - txn.execute(sql, (user_id, room_id)) + txn.execute(sql, (user_id, room_id, stream_ordering)) row = txn.fetchone() notify_count = row[0] if row else 0 @@ -155,10 +148,10 @@ class EventPushActionsWorkerStore(SQLBaseStore): " highlight = 1" " AND user_id = ?" " AND room_id = ?" - " AND %s" - ) % (lower_bound(token, self.database_engine, inclusive=False),) + " AND stream_ordering > ?" + ) - txn.execute(sql, (user_id, room_id)) + txn.execute(sql, (user_id, room_id, stream_ordering)) row = txn.fetchone() highlight_count = row[0] if row else 0 @@ -209,7 +202,6 @@ class EventPushActionsWorkerStore(SQLBaseStore): " ep.highlight " " FROM (" " SELECT room_id," - " MAX(topological_ordering) as topological_ordering," " MAX(stream_ordering) as stream_ordering" " FROM events" " INNER JOIN receipts_linearized USING (room_id, event_id)" @@ -219,13 +211,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): " event_push_actions AS ep" " WHERE" " ep.room_id = rl.room_id" - " AND (" - " ep.topological_ordering > rl.topological_ordering" - " OR (" - " ep.topological_ordering = rl.topological_ordering" - " AND ep.stream_ordering > rl.stream_ordering" - " )" - " )" + " AND ep.stream_ordering > rl.stream_ordering" " AND ep.user_id = ?" " AND ep.stream_ordering > ?" " AND ep.stream_ordering <= ?" @@ -318,7 +304,6 @@ class EventPushActionsWorkerStore(SQLBaseStore): " ep.highlight, e.received_ts" " FROM (" " SELECT room_id," - " MAX(topological_ordering) as topological_ordering," " MAX(stream_ordering) as stream_ordering" " FROM events" " INNER JOIN receipts_linearized USING (room_id, event_id)" @@ -329,13 +314,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): " INNER JOIN events AS e USING (room_id, event_id)" " WHERE" " ep.room_id = rl.room_id" - " AND (" - " ep.topological_ordering > rl.topological_ordering" - " OR (" - " ep.topological_ordering = rl.topological_ordering" - " AND ep.stream_ordering > rl.stream_ordering" - " )" - " )" + " AND ep.stream_ordering > rl.stream_ordering" " AND ep.user_id = ?" " AND ep.stream_ordering > ?" " AND ep.stream_ordering <= ?" @@ -762,10 +741,10 @@ class EventPushActionsStore(EventPushActionsWorkerStore): ) def _remove_old_push_actions_before_txn(self, txn, room_id, user_id, - topological_ordering, stream_ordering): + stream_ordering): """ Purges old push actions for a user and room before a given - topological_ordering. + stream_ordering. We however keep a months worth of highlighted notifications, so that users can still get a list of recent highlights. @@ -774,7 +753,7 @@ class EventPushActionsStore(EventPushActionsWorkerStore): txn: The transcation room_id: Room ID to delete from user_id: user ID to delete for - topological_ordering: The lowest topological ordering which will + stream_ordering: The lowest stream ordering which will not be deleted. """ txn.call_after( @@ -793,9 +772,9 @@ class EventPushActionsStore(EventPushActionsWorkerStore): txn.execute( "DELETE FROM event_push_actions " " WHERE user_id = ? AND room_id = ? AND " - " topological_ordering <= ?" + " stream_ordering <= ?" " AND ((stream_ordering < ? AND highlight = 1) or highlight = 0)", - (user_id, room_id, topological_ordering, self.stream_ordering_month_ago) + (user_id, room_id, stream_ordering, self.stream_ordering_month_ago) ) txn.execute(""" diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index 63997ed449..2f95e7e82a 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -407,7 +407,6 @@ class ReceiptsStore(ReceiptsWorkerStore): txn, room_id=room_id, user_id=user_id, - topological_ordering=topological_ordering, stream_ordering=stream_ordering, ) diff --git a/tests/storage/test_event_push_actions.py b/tests/storage/test_event_push_actions.py index 9962ce8a5d..3cbf9a78b1 100644 --- a/tests/storage/test_event_push_actions.py +++ b/tests/storage/test_event_push_actions.py @@ -55,7 +55,7 @@ class EventPushActionsStoreTestCase(tests.unittest.TestCase): def _assert_counts(noitf_count, highlight_count): counts = yield self.store.runInteraction( "", self.store._get_unread_counts_by_pos_txn, - room_id, user_id, 0, 0 + room_id, user_id, 0 ) self.assertEquals( counts, @@ -86,7 +86,7 @@ class EventPushActionsStoreTestCase(tests.unittest.TestCase): def _mark_read(stream, depth): return self.store.runInteraction( "", self.store._remove_old_push_actions_before_txn, - room_id, user_id, depth, stream + room_id, user_id, stream ) yield _assert_counts(0, 0) From f077e97914c9b5c82c94786130d98af52516cde0 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 14 May 2018 13:50:58 +0100 Subject: [PATCH 149/310] instead of inserting user daily visit data at the end of the day, instead insert incrementally through the day --- synapse/app/homeserver.py | 19 ++++++++++--- synapse/storage/__init__.py | 54 ++++++++++--------------------------- 2 files changed, 30 insertions(+), 43 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f785a7a22b..bfc79a5e81 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -17,6 +17,7 @@ import gc import logging import os import sys +import datetime import synapse import synapse.config.logger @@ -475,9 +476,24 @@ def run(hs): " changes across releases." ) + # def recurring_user_daily_visit_stats(): + def generate_user_daily_visit_stats(): hs.get_datastore().generate_user_daily_visits() + # Since user daily stats are bucketed at midnight UTC, + # and user_ips.last_seen can be updated at any time, it is important to call + # generate_user_daily_visit_stats immediately prior to the day end. Assuming + # an hourly cadence, the simplist way is to allign all calls to the hour + # end + end_of_hour = datetime.datetime.now().replace(microsecond=0, second=0, minute=0) \ + + datetime.timedelta(hours=1) \ + - datetime.timedelta(seconds=10) # Ensure method fires before day transistion + + time_to_next_hour = end_of_hour - datetime.datetime.now() + clock.call_later(time_to_next_hour.seconds, + clock.looping_call(generate_user_daily_visit_stats, 60 * 60 * 1000)) + if hs.config.report_stats: logger.info("Scheduling stats reporting for 3 hour intervals") clock.looping_call(phone_stats_home, 3 * 60 * 60 * 1000) @@ -490,9 +506,6 @@ def run(hs): # be quite busy the first few minutes clock.call_later(5 * 60, phone_stats_home) - clock.looping_call(generate_user_daily_visit_stats, 10 * 60 * 1000) - clock.call_later(5 * 60, generate_user_daily_visit_stats) - if hs.config.daemonize and hs.config.print_pidfile: print (hs.config.pid_file) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index b51cf70336..6949876c13 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -353,48 +353,22 @@ class DataStore(RoomMemberStore, RoomStore, Generates daily visit data for use in cohort/ retention analysis """ def _generate_user_daily_visits(txn): - logger.info("Calling _generate_user_daily_visits") - # determine timestamp of previous days - yesterday = datetime.datetime.utcnow() - datetime.timedelta(days=1) - yesterday_start = datetime.datetime(yesterday.year, yesterday.month, - yesterday.day, tzinfo=tz.tzutc()) - yesterday_start_time = int(time.mktime(yesterday_start.timetuple())) * 1000 - # Check that this job has not already been completed + # determine timestamp of the day start + now = datetime.datetime.utcnow() + today_start = datetime.datetime(now.year, now.month, + now.day, tzinfo=tz.tzutc()) + today_start_time = int(time.mktime(today_start.timetuple())) * 1000 + logger.info(today_start_time) sql = """ - SELECT timestamp - FROM user_daily_visits - ORDER by timestamp desc limit 1 - """ - txn.execute(sql) - row = txn.fetchone() - - # Bail if the most recent time is yesterday - if row and row[0] == yesterday_start_time: - return - - # Not specificying an upper bound means that if the update is run at - # 10 mins past midnight and the user is active during a 30 min session - # that the user is still included in the previous days stats - # This does mean that if the update is run hours late, then it is possible - # to overstate the cohort, but this seems a reasonable trade off - # The alternative is to insert on every request - but prefer to avoid - # for performance reasons - sql = """ - SELECT user_id, device_id - FROM user_ips - WHERE last_seen > ? - """ - txn.execute(sql, (yesterday_start_time,)) - user_visits = txn.fetchall() - - sql = """ - INSERT INTO user_daily_visits (user_id, device_id, timestamp) - VALUES (?, ?, ?) - """ - - for visit in user_visits: - txn.execute(sql, (visit + (yesterday_start_time,))) + INSERT INTO user_daily_visits (user_id, device_id, timestamp) + SELECT user_id, device_id, ? + FROM user_ips AS u + LEFT JOIN user_daily_visits USING (user_id, device_id) + WHERE last_seen > ? AND timestamp IS NULL + GROUP BY user_id, device_id; + """ + txn.execute(sql, (today_start_time, today_start_time)) return self.runInteraction("generate_user_daily_visits", _generate_user_daily_visits) From aea80a0118270807feb6e679d3d65562e9c9e512 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 14 May 2018 15:50:57 +0100 Subject: [PATCH 150/310] v0.29.0-rc1: bump version and change log --- CHANGES.rst | 66 +++++++++++++++++++++++++++++++++++++++++++-- synapse/__init__.py | 2 +- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 9d40b2ac1e..a5c3848bce 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,9 @@ -Changes in synapse -=============================== +Changes in synapse v0.29.0-rc1 (2018-05-14) +=========================================== + +Notable changes, a docker file for running Synapse (Thanks to @kaiyou!) and a +closed spec bug in the Client Server API. Additionally further prep for Python 3 +migration. Potentially breaking change: @@ -12,6 +16,64 @@ Potentially breaking change: Thanks to @NotAFile for fixing this. +Features: + +* Add a Dockerfile for synapse (PR #2846) Thanks to @kaiyou! + +Changes: + +* nuke-room-from-db.sh: added postgresql option and help (PR #2337) Thanks to @rubo77! +* Make 'unexpected logging context' into warnings (PR #3007) +* Open certificate files as bytes (PR #3084) Thanks to @NotAFile! +* Open config file in non-bytes mode (PR #3085) Thanks to @NotAFile! +* Make event properties raise AttributeError instead (PR #3102) Thanks to @NotAFile! +* Use six.moves.urlparse (PR #3108) Thanks to @NotAFile! +* Use deferred.addTimeout instead of time_bound_deferred (PR #3127) +* remove duplicates from groups tables (PR #3129) +* Miscellaneous fixes to python_dependencies (PR #3136) +* Improve exception handling for background processes (PR #3138) +* Add missing consumeErrors to improve exception handling (PR #3139) +* Use run_in_background in preference to preserve_fn (PR #3140) +* Refactor event storage to prepare for changes in state calculations (PR #3141) +* reraise exceptions more carefully (PR #3142) +* Remove redundant call to preserve_fn (PR #3143) +* Trap exceptions thrown within run_in_background (PR #3144) +* Add py3 tests to tox with folders that work (PR #3145) Thanks to @NotAFile! +* Don't yield in list comprehensions (PR #3150) Thanks to @NotAFile! +* Move more xrange to six (PR #3151) Thanks to @NotAFile! +* make imports local (PR #3152) Thanks to @NotAFile! +* move httplib import to six (PR #3153) Thanks to @NotAFile! +* Replace stringIO imports with six (PR #3154) Thanks to @NotAFile! +* more bytes strings (PR #3155) Thanks to @NotAFile! +* Construct HMAC as bytes on py3 (PR #3156) Thanks to @NotAFile! +* run config tests on py3 (PR #3159) Thanks to @NotAFile! +* add guard for None on purge_history api (PR #3160) Thanks to @krombel! +* Refactor /context to reuse pagination storage functions (PR #3193) +* Refactor recent events func to use pagination func (PR #3195) +* Refactor pagination DB API to return concrete type (PR #3196) +* Refactor get_recent_events_for_room return type (PR #3198) +* Refactor sync APIs to reuse pagination API (PR #3199) +* Remove unused code path from member change DB func (PR #3200) +* Part user from rooms on account deactivate (PR #3201) +* Refactor request handling wrappers (PR #3203) +* transaction_id, destination defined twice (PR #3209) Thanks to @damir-manapov! + +Bug Fixes: + +* synapse fails to start under Twisted >= 18.4 (PR #3135) +* Fix incorrect reference to StringIO (PR #3168) +* Fix a class of logcontext leaks (PR #3170) +* Fix a couple of logcontext leaks in unit tests (PR #3172) +* Fix logcontext leak in media repo (PR #3174) +* Escape label values in prometheus metrics (PR #3175) +* fix http request timeout code (PR #3178) +* Fix 'Unhandled Error' logs with Twisted 18.4 (PR #3182) Thanks to @Half-Shot! +* Fix logcontext leaks in rate limiter (PR #3183) +* Fix metrics that have integer value labels (PR #3186) +* notifications: Convert next_token to string according to the spec (PR #3190) Thanks to @mujx! +* nuke-room-from-db.sh: fix deletion from search table (PR #3194) Thanks to @rubo77! +* Set Server header in SynapseRequest (PR #3208) + Changes in synapse v0.28.1 (2018-05-01) ======================================= diff --git a/synapse/__init__.py b/synapse/__init__.py index f31cb9a3cb..69c081eb37 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.28.1" +__version__ = "0.29.0-rc1" From e71fb118f4d57f0c77647aa6fcabe36c08e31b20 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 14 May 2018 17:39:22 +0100 Subject: [PATCH 151/310] rearrange and collect related PRs --- CHANGES.rst | 53 +++++++++++++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index a5c3848bce..aed8525f2d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -20,60 +20,61 @@ Features: * Add a Dockerfile for synapse (PR #2846) Thanks to @kaiyou! -Changes: +Changes - General: * nuke-room-from-db.sh: added postgresql option and help (PR #2337) Thanks to @rubo77! +* Part user from rooms on account deactivate (PR #3201) * Make 'unexpected logging context' into warnings (PR #3007) -* Open certificate files as bytes (PR #3084) Thanks to @NotAFile! -* Open config file in non-bytes mode (PR #3085) Thanks to @NotAFile! -* Make event properties raise AttributeError instead (PR #3102) Thanks to @NotAFile! -* Use six.moves.urlparse (PR #3108) Thanks to @NotAFile! -* Use deferred.addTimeout instead of time_bound_deferred (PR #3127) +* Set Server header in SynapseRequest (PR #3208) +* Use deferred.addTimeout instead of time_bound_deferred (PR #3127, #3178) * remove duplicates from groups tables (PR #3129) -* Miscellaneous fixes to python_dependencies (PR #3136) * Improve exception handling for background processes (PR #3138) * Add missing consumeErrors to improve exception handling (PR #3139) * Use run_in_background in preference to preserve_fn (PR #3140) -* Refactor event storage to prepare for changes in state calculations (PR #3141) * reraise exceptions more carefully (PR #3142) * Remove redundant call to preserve_fn (PR #3143) * Trap exceptions thrown within run_in_background (PR #3144) -* Add py3 tests to tox with folders that work (PR #3145) Thanks to @NotAFile! -* Don't yield in list comprehensions (PR #3150) Thanks to @NotAFile! -* Move more xrange to six (PR #3151) Thanks to @NotAFile! -* make imports local (PR #3152) Thanks to @NotAFile! -* move httplib import to six (PR #3153) Thanks to @NotAFile! -* Replace stringIO imports with six (PR #3154) Thanks to @NotAFile! -* more bytes strings (PR #3155) Thanks to @NotAFile! -* Construct HMAC as bytes on py3 (PR #3156) Thanks to @NotAFile! -* run config tests on py3 (PR #3159) Thanks to @NotAFile! -* add guard for None on purge_history api (PR #3160) Thanks to @krombel! + +Changes - Refactors: + * Refactor /context to reuse pagination storage functions (PR #3193) * Refactor recent events func to use pagination func (PR #3195) * Refactor pagination DB API to return concrete type (PR #3196) * Refactor get_recent_events_for_room return type (PR #3198) * Refactor sync APIs to reuse pagination API (PR #3199) * Remove unused code path from member change DB func (PR #3200) -* Part user from rooms on account deactivate (PR #3201) * Refactor request handling wrappers (PR #3203) * transaction_id, destination defined twice (PR #3209) Thanks to @damir-manapov! +* Refactor event storage to prepare for changes in state calculations (PR #3141) + +Changes - Python 3 migration: + +* Construct HMAC as bytes on py3 (PR #3156) Thanks to @NotAFile! +* run config tests on py3 (PR #3159) Thanks to @NotAFile! +* Open certificate files as bytes (PR #3084) Thanks to @NotAFile! +* Open config file in non-bytes mode (PR #3085) Thanks to @NotAFile! +* Make event properties raise AttributeError instead (PR #3102) Thanks to @NotAFile! +* Use six.moves.urlparse (PR #3108) Thanks to @NotAFile! +* Add py3 tests to tox with folders that work (PR #3145) Thanks to @NotAFile! +* Don't yield in list comprehensions (PR #3150) Thanks to @NotAFile! +* Move more xrange to six (PR #3151) Thanks to @NotAFile! +* make imports local (PR #3152) Thanks to @NotAFile! +* move httplib import to six (PR #3153) Thanks to @NotAFile! +* Replace stringIO imports with six (PR #3154, #3168) Thanks to @NotAFile! +* more bytes strings (PR #3155) Thanks to @NotAFile! Bug Fixes: -* synapse fails to start under Twisted >= 18.4 (PR #3135) -* Fix incorrect reference to StringIO (PR #3168) +* synapse fails to start under Twisted >= 18.4 (PR #3157) * Fix a class of logcontext leaks (PR #3170) * Fix a couple of logcontext leaks in unit tests (PR #3172) * Fix logcontext leak in media repo (PR #3174) -* Escape label values in prometheus metrics (PR #3175) -* fix http request timeout code (PR #3178) +* Escape label values in prometheus metrics (PR #3175, #3186) * Fix 'Unhandled Error' logs with Twisted 18.4 (PR #3182) Thanks to @Half-Shot! * Fix logcontext leaks in rate limiter (PR #3183) -* Fix metrics that have integer value labels (PR #3186) * notifications: Convert next_token to string according to the spec (PR #3190) Thanks to @mujx! * nuke-room-from-db.sh: fix deletion from search table (PR #3194) Thanks to @rubo77! -* Set Server header in SynapseRequest (PR #3208) - +* add guard for None on purge_history api (PR #3160) Thanks to @krombel! Changes in synapse v0.28.1 (2018-05-01) ======================================= From 589ecc5b5800a86406a0d1464fd6974c029a1163 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 14 May 2018 17:49:59 +0100 Subject: [PATCH 152/310] further musical chairs --- CHANGES.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index aed8525f2d..78d015db9a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -26,11 +26,9 @@ Changes - General: * Part user from rooms on account deactivate (PR #3201) * Make 'unexpected logging context' into warnings (PR #3007) * Set Server header in SynapseRequest (PR #3208) -* Use deferred.addTimeout instead of time_bound_deferred (PR #3127, #3178) * remove duplicates from groups tables (PR #3129) * Improve exception handling for background processes (PR #3138) * Add missing consumeErrors to improve exception handling (PR #3139) -* Use run_in_background in preference to preserve_fn (PR #3140) * reraise exceptions more carefully (PR #3142) * Remove redundant call to preserve_fn (PR #3143) * Trap exceptions thrown within run_in_background (PR #3144) @@ -46,6 +44,9 @@ Changes - Refactors: * Refactor request handling wrappers (PR #3203) * transaction_id, destination defined twice (PR #3209) Thanks to @damir-manapov! * Refactor event storage to prepare for changes in state calculations (PR #3141) +* Set Server header in SynapseRequest (PR #3208) +* Use deferred.addTimeout instead of time_bound_deferred (PR #3127, #3178) +* Use run_in_background in preference to preserve_fn (PR #3140) Changes - Python 3 migration: From 47815edcfae73c5b938f8354853a09c0b80ef27e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 11 May 2018 00:17:11 +0100 Subject: [PATCH 153/310] ConsentResource to gather policy consent from users Hopefully there are enough comments and docs in this that it makes sense on its own. --- docs/privacy_policy_templates/README.md | 23 ++ docs/privacy_policy_templates/en/1.0.html | 17 ++ docs/privacy_policy_templates/en/success.html | 11 + synapse/app/homeserver.py | 9 + synapse/config/__init__.py | 6 + synapse/config/consent_config.py | 42 ++++ synapse/config/homeserver.py | 8 +- synapse/config/key.py | 10 + synapse/http/server.py | 76 ++++++- synapse/rest/consent/__init__.py | 0 synapse/rest/consent/consent_resource.py | 210 ++++++++++++++++++ synapse/server.py | 3 + synapse/storage/registration.py | 18 ++ .../schema/delta/48/add_user_consent.sql | 18 ++ 14 files changed, 446 insertions(+), 5 deletions(-) create mode 100644 docs/privacy_policy_templates/README.md create mode 100644 docs/privacy_policy_templates/en/1.0.html create mode 100644 docs/privacy_policy_templates/en/success.html create mode 100644 synapse/config/consent_config.py create mode 100644 synapse/rest/consent/__init__.py create mode 100644 synapse/rest/consent/consent_resource.py create mode 100644 synapse/storage/schema/delta/48/add_user_consent.sql diff --git a/docs/privacy_policy_templates/README.md b/docs/privacy_policy_templates/README.md new file mode 100644 index 0000000000..8e91c516b3 --- /dev/null +++ b/docs/privacy_policy_templates/README.md @@ -0,0 +1,23 @@ +If enabling the 'consent' resource in synapse, you will need some templates +for the HTML to be served to the user. This directory contains very simple +examples of the sort of thing that can be done. + +You'll need to add this sort of thing to your homeserver.yaml: + +``` +form_secret: + +user_consent: + template_dir: docs/privacy_policy_templates + default_version: 1.0 +``` + +You should then be able to enable the `consent` resource under a `listener` +entry. For example: + +``` +listeners: + - port: 8008 + resources: + - names: [client, consent] +``` diff --git a/docs/privacy_policy_templates/en/1.0.html b/docs/privacy_policy_templates/en/1.0.html new file mode 100644 index 0000000000..ab8666f0c3 --- /dev/null +++ b/docs/privacy_policy_templates/en/1.0.html @@ -0,0 +1,17 @@ + + + + Matrix.org Privacy policy + + +

+ All your base are belong to us. +

+
+ + + + +
+ + diff --git a/docs/privacy_policy_templates/en/success.html b/docs/privacy_policy_templates/en/success.html new file mode 100644 index 0000000000..d55e90c94f --- /dev/null +++ b/docs/privacy_policy_templates/en/success.html @@ -0,0 +1,11 @@ + + + + Matrix.org Privacy policy + + +

+ Sweet. +

+ + diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index a0e465d644..730271628e 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -41,6 +41,7 @@ from synapse.python_dependencies import CONDITIONAL_REQUIREMENTS, \ from synapse.replication.http import ReplicationRestResource, REPLICATION_PREFIX from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory from synapse.rest import ClientRestResource +from synapse.rest.consent.consent_resource import ConsentResource from synapse.rest.key.v1.server_key_resource import LocalKey from synapse.rest.key.v2 import KeyApiV2Resource from synapse.rest.media.v0.content_repository import ContentRepoResource @@ -182,6 +183,14 @@ class SynapseHomeServer(HomeServer): "/_matrix/client/versions": client_resource, }) + if name == "consent": + consent_resource = ConsentResource(self) + if compress: + consent_resource = gz_wrap(consent_resource) + resources.update({ + "/_matrix/consent": consent_resource, + }) + if name == "federation": resources.update({ FEDERATION_PREFIX: TransportLayerServer(self), diff --git a/synapse/config/__init__.py b/synapse/config/__init__.py index bfebb0f644..f2a5a41e92 100644 --- a/synapse/config/__init__.py +++ b/synapse/config/__init__.py @@ -12,3 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from ._base import ConfigError + +# export ConfigError if somebody does import * +# this is largely a fudge to stop PEP8 moaning about the import +__all__ = ["ConfigError"] diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py new file mode 100644 index 0000000000..675fce0911 --- /dev/null +++ b/synapse/config/consent_config.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import Config + +DEFAULT_CONFIG = """\ +# User Consent configuration +# +# uncomment and configure if enabling the 'consent' resource under 'listeners'. +# +# 'template_dir' gives the location of the templates for the HTML forms. +# This directory should contain one subdirectory per language (eg, 'en', 'fr'), +# and each language directory should contain the policy document (named as +# '.html') and a success page (success.html). +# +# 'default_version' gives the version of the policy document to serve up if +# there is no 'v' parameter. +# +# user_consent: +# template_dir: res/templates/privacy +# default_version: 1.0 +""" + + +class ConsentConfig(Config): + def read_config(self, config): + self.consent_config = config.get("user_consent") + + def default_config(self, **kwargs): + return DEFAULT_CONFIG diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index bf19cfee29..fb6bd3b421 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,7 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - from .tls import TlsConfig from .server import ServerConfig from .logger import LoggingConfig @@ -37,6 +37,7 @@ from .push import PushConfig from .spam_checker import SpamCheckerConfig from .groups import GroupsConfig from .user_directory import UserDirectoryConfig +from .consent_config import ConsentConfig class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, @@ -45,12 +46,13 @@ class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, AppServiceConfig, KeyConfig, SAML2Config, CasConfig, JWTConfig, PasswordConfig, EmailConfig, WorkerConfig, PasswordAuthProviderConfig, PushConfig, - SpamCheckerConfig, GroupsConfig, UserDirectoryConfig,): + SpamCheckerConfig, GroupsConfig, UserDirectoryConfig, + ConsentConfig): pass if __name__ == '__main__': import sys sys.stdout.write( - HomeServerConfig().generate_config(sys.argv[1], sys.argv[2])[0] + HomeServerConfig().generate_config(sys.argv[1], sys.argv[2], True)[0] ) diff --git a/synapse/config/key.py b/synapse/config/key.py index 4b8fc063d0..d1382ad9ac 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -59,14 +59,20 @@ class KeyConfig(Config): self.expire_access_token = config.get("expire_access_token", False) + # a secret which is used to calculate HMACs for form values, to stop + # falsification of values + self.form_secret = config.get("form_secret", None) + def default_config(self, config_dir_path, server_name, is_generating_file=False, **kwargs): base_key_name = os.path.join(config_dir_path, server_name) if is_generating_file: macaroon_secret_key = random_string_with_symbols(50) + form_secret = '"%s"' % random_string_with_symbols(50) else: macaroon_secret_key = None + form_secret = 'null' return """\ macaroon_secret_key: "%(macaroon_secret_key)s" @@ -74,6 +80,10 @@ class KeyConfig(Config): # Used to enable access token expiration. expire_access_token: False + # a secret which is used to calculate HMACs for form values, to stop + # falsification of values + form_secret: %(form_secret)s + ## Signing Keys ## # Path to the signing key to sign messages with diff --git a/synapse/http/server.py b/synapse/http/server.py index f29e36f490..a38209770d 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -13,7 +13,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import cgi +from six.moves import http_client from synapse.api.errors import ( cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError, Codes @@ -44,6 +45,18 @@ import simplejson logger = logging.getLogger(__name__) +HTML_ERROR_TEMPLATE = """ + + + + Error {code} + + +

{msg}

+ + +""" + def wrap_json_request_handler(h): """Wraps a request handler method with exception handling. @@ -104,6 +117,65 @@ def wrap_json_request_handler(h): return wrap_request_handler_with_logging(wrapped_request_handler) +def wrap_html_request_handler(h): + """Wraps a request handler method with exception handling. + + Also adds logging as per wrap_request_handler_with_logging. + + The handler method must have a signature of "handle_foo(self, request)", + where "self" must have a "clock" attribute (and "request" must be a + SynapseRequest). + """ + def wrapped_request_handler(self, request): + d = defer.maybeDeferred(h, self, request) + d.addErrback(_return_html_error, request) + return d + + return wrap_request_handler_with_logging(wrapped_request_handler) + + +def _return_html_error(f, request): + """Sends an HTML error page corresponding to the given failure + + Args: + f (twisted.python.failure.Failure): + request (twisted.web.iweb.IRequest): + """ + if f.check(CodeMessageException): + cme = f.value + code = cme.code + msg = cme.msg + + if isinstance(cme, SynapseError): + logger.info( + "%s SynapseError: %s - %s", request, code, msg + ) + else: + logger.error( + "Failed handle request %r: %s", + request, + f.getTraceback().rstrip(), + ) + else: + code = http_client.INTERNAL_SERVER_ERROR + msg = "Internal server error" + + logger.error( + "Failed handle request %r: %s", + request, + f.getTraceback().rstrip(), + ) + + body = HTML_ERROR_TEMPLATE.format( + code=code, msg=cgi.escape(msg), + ).encode("utf-8") + request.setResponseCode(code) + request.setHeader(b"Content-Type", b"text/html; charset=utf-8") + request.setHeader(b"Content-Length", b"%i" % (len(body),)) + request.write(body) + finish_request(request) + + def wrap_request_handler_with_logging(h): """Wraps a request handler to provide logging and metrics @@ -134,7 +206,7 @@ def wrap_request_handler_with_logging(h): servlet_name = self.__class__.__name__ with request.processing(servlet_name): with PreserveLoggingContext(request_context): - d = h(self, request) + d = defer.maybeDeferred(h, self, request) # record the arrival of the request *after* # dispatching to the handler, so that the handler diff --git a/synapse/rest/consent/__init__.py b/synapse/rest/consent/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py new file mode 100644 index 0000000000..d791302278 --- /dev/null +++ b/synapse/rest/consent/consent_resource.py @@ -0,0 +1,210 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from hashlib import sha256 +import hmac +import logging +from os import path +from six.moves import http_client + +import jinja2 +from jinja2 import TemplateNotFound +from twisted.internet import defer +from twisted.web.resource import Resource +from twisted.web.server import NOT_DONE_YET + +from synapse.api.errors import NotFoundError, SynapseError, StoreError +from synapse.config import ConfigError +from synapse.http.server import ( + finish_request, + wrap_html_request_handler, +) +from synapse.http.servlet import parse_string +from synapse.types import UserID + + +# language to use for the templates. TODO: figure this out from Accept-Language +TEMPLATE_LANGUAGE = "en" + +logger = logging.getLogger(__name__) + +# use hmac.compare_digest if we have it (python 2.7.7), else just use equality +if hasattr(hmac, "compare_digest"): + compare_digest = hmac.compare_digest +else: + def compare_digest(a, b): + return a == b + + +class ConsentResource(Resource): + """A twisted Resource to display a privacy policy and gather consent to it + + When accessed via GET, returns the privacy policy via a template. + + When accessed via POST, records the user's consent in the database and + displays a success page. + + The config should include a template_dir setting which contains templates + for the HTML. The directory should contain one subdirectory per language + (eg, 'en', 'fr'), and each language directory should contain the policy + document (named as '.html') and a success page (success.html). + + Both forms take a set of parameters from the browser. For the POST form, + these are normally sent as form parameters (but may be query-params); for + GET requests they must be query params. These are: + + u: the complete mxid, or the localpart of the user giving their + consent. Required for both GET (where it is used as an input to the + template) and for POST (where it is used to find the row in the db + to update). + + h: hmac_sha256(secret, u), where 'secret' is the privacy_secret in the + config file. If it doesn't match, the request is 403ed. + + v: the version of the privacy policy being agreed to. + + For GET: optional, and defaults to whatever was set in the config + file. Used to choose the version of the policy to pick from the + templates directory. + + For POST: required; gives the value to be recorded in the database + against the user. + """ + def __init__(self, hs): + """ + Args: + hs (synapse.server.HomeServer): homeserver + """ + Resource.__init__(self) + + self.hs = hs + self.store = hs.get_datastore() + + # this is required by the request_handler wrapper + self.clock = hs.get_clock() + + consent_config = hs.config.consent_config + if consent_config is None: + raise ConfigError( + "Consent resource is enabled but user_consent section is " + "missing in config file.", + ) + + # daemonize changes the cwd to /, so make the path absolute now. + consent_template_directory = path.abspath( + consent_config["template_dir"], + ) + if not path.isdir(consent_template_directory): + raise ConfigError( + "Could not find template directory '%s'" % ( + consent_template_directory, + ), + ) + + loader = jinja2.FileSystemLoader(consent_template_directory) + self._jinja_env = jinja2.Environment(loader=loader) + + self._default_consent_verison = consent_config["default_version"] + + if hs.config.form_secret is None: + raise ConfigError( + "Consent resource is enabled but form_secret is not set in " + "config file. It should be set to an arbitrary secret string.", + ) + + self._hmac_secret = hs.config.form_secret.encode("utf-8") + + def render_GET(self, request): + self._async_render_GET(request) + return NOT_DONE_YET + + @wrap_html_request_handler + def _async_render_GET(self, request): + """ + Args: + request (twisted.web.http.Request): + """ + + version = parse_string(request, "v", + default=self._default_consent_verison) + username = parse_string(request, "u", required=True) + userhmac = parse_string(request, "h", required=True) + + self._check_hash(username, userhmac) + + try: + self._render_template( + request, "%s.html" % (version,), + user=username, userhmac=userhmac, version=version, + ) + except TemplateNotFound: + raise NotFoundError("Unknown policy version") + + def render_POST(self, request): + self._async_render_POST(request) + return NOT_DONE_YET + + @wrap_html_request_handler + @defer.inlineCallbacks + def _async_render_POST(self, request): + """ + Args: + request (twisted.web.http.Request): + """ + version = parse_string(request, "v", required=True) + username = parse_string(request, "u", required=True) + userhmac = parse_string(request, "h", required=True) + + self._check_hash(username, userhmac) + + if username.startswith('@'): + qualified_user_id = username + else: + qualified_user_id = UserID(username, self.hs.hostname).to_string() + + try: + yield self.store.user_set_consent_version(qualified_user_id, version) + except StoreError as e: + if e.code != 404: + raise + raise NotFoundError("Unknown user") + + try: + self._render_template(request, "success.html") + except TemplateNotFound: + raise NotFoundError("success.html not found") + + def _render_template(self, request, template_name, **template_args): + # get_template checks for ".." so we don't need to worry too much + # about path traversal here. + template_html = self._jinja_env.get_template( + path.join(TEMPLATE_LANGUAGE, template_name) + ) + html_bytes = template_html.render(**template_args).encode("utf8") + + request.setHeader(b"Content-Type", b"text/html; charset=utf-8") + request.setHeader(b"Content-Length", b"%i" % len(html_bytes)) + request.write(html_bytes) + finish_request(request) + + def _check_hash(self, userid, userhmac): + want_mac = hmac.new( + key=self._hmac_secret, + msg=userid, + digestmod=sha256, + ).hexdigest() + + if not compare_digest(want_mac, userhmac): + raise SynapseError(http_client.FORBIDDEN, "HMAC incorrect") diff --git a/synapse/server.py b/synapse/server.py index ebdea6b0c4..21cde5b6fc 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -97,6 +97,9 @@ class HomeServer(object): which must be implemented by the subclass. This code may call any of the required "get" methods on the instance to obtain the sub-dependencies that one requires. + + Attributes: + config (synapse.config.homeserver.HomeserverConfig): """ DEPENDENCIES = [ diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index a50717db2d..6ffc397861 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -286,6 +286,24 @@ class RegistrationStore(RegistrationWorkerStore, "user_set_password_hash", user_set_password_hash_txn ) + def user_set_consent_version(self, user_id, consent_version): + """Updates the user table to record privacy policy consent + + Args: + user_id (str): full mxid of the user to update + consent_version (str): version of the policy the user has consented + to + + Raises: + StoreError(404) if user not found + """ + return self._simple_update_one( + table='users', + keyvalues={'name': user_id, }, + updatevalues={'consent_version': consent_version, }, + desc="user_set_consent_version" + ) + def user_delete_access_tokens(self, user_id, except_token_id=None, device_id=None): """ diff --git a/synapse/storage/schema/delta/48/add_user_consent.sql b/synapse/storage/schema/delta/48/add_user_consent.sql new file mode 100644 index 0000000000..5237491506 --- /dev/null +++ b/synapse/storage/schema/delta/48/add_user_consent.sql @@ -0,0 +1,18 @@ +/* Copyright 2018 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* record the version of the privacy policy the user has consented to + */ +ALTER TABLE users ADD COLUMN consent_version TEXT; From 37dbee6490dbb9cd66d887ed4d902ae8482fae2f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 15 May 2018 15:48:40 +0100 Subject: [PATCH 154/310] Use events_to_purge table rather than token --- synapse/storage/events.py | 49 +++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 05cde96afc..b4ae6664f0 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -1871,6 +1871,10 @@ class EventsStore(EventsWorkerStore): "CREATE INDEX events_to_purge_should_delete" " ON events_to_purge(should_delete)", ) + txn.execute( + "CREATE INDEX events_to_purge_id" + " ON events_to_purge(event_id)", + ) # First ensure that we're not about to delete all the forward extremeties txn.execute( @@ -1927,9 +1931,8 @@ class EventsStore(EventsWorkerStore): txn.execute( "SELECT DISTINCT e.event_id FROM events_to_purge AS e" " INNER JOIN event_edges AS ed ON e.event_id = ed.prev_event_id" - " INNER JOIN events AS e2 ON e2.event_id = ed.event_id" - " WHERE e2.topological_ordering >= ?", - (topological_ordering, ) + " LEFT JOIN events_to_purge AS ep2 ON ed.event_id = ep2.event_id" + " WHERE ep2.event_id IS NULL", ) new_backwards_extrems = txn.fetchall() @@ -1953,16 +1956,22 @@ class EventsStore(EventsWorkerStore): # Get all state groups that are only referenced by events that are # to be deleted. - txn.execute( - "SELECT state_group FROM event_to_state_groups" - " INNER JOIN events USING (event_id)" - " WHERE state_group IN (" - " SELECT DISTINCT state_group FROM events_to_purge" - " INNER JOIN event_to_state_groups USING (event_id)" - " )" - " GROUP BY state_group HAVING MAX(topological_ordering) < ?", - (topological_ordering, ) - ) + # This works by first getting state groups that we may want to delete, + # joining against event_to_state_groups to get events that use that + # state group, then left joining against events_to_purge again. Any + # state group where the left join produce *no nulls* are referenced + # only by events that are going to be purged. + txn.execute(""" + SELECT state_group FROM + ( + SELECT DISTINCT state_group FROM events_to_purge + INNER JOIN event_to_state_groups USING (event_id) + ) AS sp + INNER JOIN event_to_state_groups USING (state_group) + LEFT JOIN events_to_purge AS ep USING (event_id) + GROUP BY state_group + HAVING SUM(CASE WHEN ep.event_id IS NULL THEN 1 ELSE 0 END) = 0 + """) state_rows = txn.fetchall() logger.info("[purge] found %i redundant state groups", len(state_rows)) @@ -2109,10 +2118,20 @@ class EventsStore(EventsWorkerStore): # # So, let's stick it at the end so that we don't block event # persistence. - logger.info("[purge] updating room_depth") + txn.execute(""" + SELECT COALESCE(MIN(depth), 0) + FROM event_backward_extremities AS eb + INNER JOIN event_edges AS eg ON eg.prev_event_id = eb.event_id + INNER JOIN events AS e ON e.event_id = eg.event_id + WHERE eb.room_id = ? + """, (room_id,)) + min_depth, = txn.fetchone() + + logger.info("[purge] updating room_depth to %d", min_depth) + txn.execute( "UPDATE room_depth SET min_depth = ? WHERE room_id = ?", - (topological_ordering, room_id,) + (min_depth, room_id,) ) # finally, drop the temp table. this will commit the txn in sqlite, From 5f27ed75ad804ab9b5287f0deb8fd24b9a3a6232 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 15 May 2018 16:06:30 +0100 Subject: [PATCH 155/310] Make purge_history operate on tokens As we're soon going to change how topological_ordering works --- synapse/handlers/message.py | 12 ++++++------ synapse/rest/client/v1/admin.py | 17 ++++++++++------- synapse/storage/events.py | 17 +++++++++-------- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index b793fc4df7..8343b5839d 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -86,14 +86,14 @@ class MessageHandler(BaseHandler): # map from purge id to PurgeStatus self._purges_by_id = {} - def start_purge_history(self, room_id, topological_ordering, + def start_purge_history(self, room_id, token, delete_local_events=False): """Start off a history purge on a room. Args: room_id (str): The room to purge from - topological_ordering (int): minimum topo ordering to preserve + token (str): topological token to delete events before delete_local_events (bool): True to delete local events as well as remote ones @@ -115,19 +115,19 @@ class MessageHandler(BaseHandler): self._purges_by_id[purge_id] = PurgeStatus() run_in_background( self._purge_history, - purge_id, room_id, topological_ordering, delete_local_events, + purge_id, room_id, token, delete_local_events, ) return purge_id @defer.inlineCallbacks - def _purge_history(self, purge_id, room_id, topological_ordering, + def _purge_history(self, purge_id, room_id, token, delete_local_events): """Carry out a history purge on a room. Args: purge_id (str): The id for this purge room_id (str): The room to purge from - topological_ordering (int): minimum topo ordering to preserve + token (str): topological token to delete events before delete_local_events (bool): True to delete local events as well as remote ones @@ -138,7 +138,7 @@ class MessageHandler(BaseHandler): try: with (yield self.pagination_lock.write(room_id)): yield self.store.purge_history( - room_id, topological_ordering, delete_local_events, + room_id, token, delete_local_events, ) logger.info("[purge] complete") self._purges_by_id[purge_id].status = PurgeStatus.STATUS_COMPLETE diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index efd5c9873d..282ce6be42 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -151,10 +151,11 @@ class PurgeHistoryRestServlet(ClientV1RestServlet): if event.room_id != room_id: raise SynapseError(400, "Event is for wrong room.") - depth = event.depth + token = yield self.store.get_topological_token_for_event(event_id) + logger.info( - "[purge] purging up to depth %i (event_id %s)", - depth, event_id, + "[purge] purging up to token %s (event_id %s)", + token, event_id, ) elif 'purge_up_to_ts' in body: ts = body['purge_up_to_ts'] @@ -174,7 +175,9 @@ class PurgeHistoryRestServlet(ClientV1RestServlet): ) ) if room_event_after_stream_ordering: - (_, depth, _) = room_event_after_stream_ordering + token = yield self.store.get_topological_token_for_event( + room_event_after_stream_ordering, + ) else: logger.warn( "[purge] purging events not possible: No event found " @@ -187,9 +190,9 @@ class PurgeHistoryRestServlet(ClientV1RestServlet): errcode=Codes.NOT_FOUND, ) logger.info( - "[purge] purging up to depth %i (received_ts %i => " + "[purge] purging up to token %d (received_ts %i => " "stream_ordering %i)", - depth, ts, stream_ordering, + token, ts, stream_ordering, ) else: raise SynapseError( @@ -199,7 +202,7 @@ class PurgeHistoryRestServlet(ClientV1RestServlet): ) purge_id = yield self.handlers.message_handler.start_purge_history( - room_id, depth, + room_id, token, delete_local_events=delete_local_events, ) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index b4ae6664f0..f65e18c1ee 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -33,7 +33,7 @@ from synapse.util.metrics import Measure from synapse.api.constants import EventTypes from synapse.api.errors import SynapseError from synapse.util.caches.descriptors import cached, cachedInlineCallbacks -from synapse.types import get_domain_from_id +from synapse.types import get_domain_from_id, RoomStreamToken import synapse.metrics # these are only included to make the type annotations work @@ -1803,15 +1803,14 @@ class EventsStore(EventsWorkerStore): return self.runInteraction("get_all_new_events", get_all_new_events_txn) def purge_history( - self, room_id, topological_ordering, delete_local_events, + self, room_id, token, delete_local_events, ): """Deletes room history before a certain point Args: room_id (str): - topological_ordering (int): - minimum topo ordering to preserve + token (str): A topological token to delete events before delete_local_events (bool): if True, we will delete local events as well as remote ones @@ -1821,12 +1820,12 @@ class EventsStore(EventsWorkerStore): return self.runInteraction( "purge_history", - self._purge_history_txn, room_id, topological_ordering, + self._purge_history_txn, room_id, token, delete_local_events, ) def _purge_history_txn( - self, txn, room_id, topological_ordering, delete_local_events, + self, txn, room_id, token, delete_local_events, ): # Tables that should be pruned: # event_auth @@ -1856,6 +1855,8 @@ class EventsStore(EventsWorkerStore): # furthermore, we might already have the table from a previous (failed) # purge attempt, so let's drop the table first. + token = RoomStreamToken.parse(token) + txn.execute("DROP TABLE IF EXISTS events_to_purge") txn.execute( @@ -1888,7 +1889,7 @@ class EventsStore(EventsWorkerStore): rows = txn.fetchall() max_depth = max(row[0] for row in rows) - if max_depth <= topological_ordering: + if max_depth <= token.topological: # We need to ensure we don't delete all the events from the datanase # otherwise we wouldn't be able to send any events (due to not # having any backwards extremeties) @@ -1904,7 +1905,7 @@ class EventsStore(EventsWorkerStore): should_delete_expr += " AND event_id NOT LIKE ?" should_delete_params += ("%:" + self.hs.hostname, ) - should_delete_params += (room_id, topological_ordering) + should_delete_params += (room_id, token.topological) txn.execute( "INSERT INTO events_to_purge" From 05ac15ae824cc538b869e3cc8db7af2ac22e6754 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Tue, 15 May 2018 17:01:33 +0100 Subject: [PATCH 156/310] Limit query load of generate_user_daily_visits The aim is to keep track of when it was last called and only query from that point in time --- synapse/app/homeserver.py | 19 ++++-------- synapse/storage/__init__.py | 60 +++++++++++++++++++++++++++++-------- 2 files changed, 53 insertions(+), 26 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index bfc79a5e81..f25eaf9ffc 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -476,23 +476,16 @@ def run(hs): " changes across releases." ) - # def recurring_user_daily_visit_stats(): - def generate_user_daily_visit_stats(): hs.get_datastore().generate_user_daily_visits() - # Since user daily stats are bucketed at midnight UTC, - # and user_ips.last_seen can be updated at any time, it is important to call - # generate_user_daily_visit_stats immediately prior to the day end. Assuming - # an hourly cadence, the simplist way is to allign all calls to the hour - # end - end_of_hour = datetime.datetime.now().replace(microsecond=0, second=0, minute=0) \ - + datetime.timedelta(hours=1) \ - - datetime.timedelta(seconds=10) # Ensure method fires before day transistion + def recurring_user_daily_visit_stats(): + clock.looping_call(generate_user_daily_visit_stats, 60 * 60 * 1000) - time_to_next_hour = end_of_hour - datetime.datetime.now() - clock.call_later(time_to_next_hour.seconds, - clock.looping_call(generate_user_daily_visit_stats, 60 * 60 * 1000)) + # Rather than update on per session basis, batch up the requests. + # If you increase the loop period, the accuracy of user_daily_visits + # table will decrease + clock.looping_call(generate_user_daily_visit_stats, 5 * 60 * 1000) if hs.config.report_stats: logger.info("Scheduling stats reporting for 3 hour intervals") diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 6949876c13..52f176a03c 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -214,6 +214,9 @@ class DataStore(RoomMemberStore, RoomStore, self._stream_order_on_start = self.get_room_max_stream_ordering() self._min_stream_order_on_start = self.get_room_min_stream_ordering() + # Used in _generate_user_daily_visits to keep track of progress + self._last_user_visit_update = self._get_start_of_day() + super(DataStore, self).__init__(db_conn, hs) def take_presence_startup_info(self): @@ -348,27 +351,58 @@ class DataStore(RoomMemberStore, RoomStore, return self.runInteraction("count_r30_users", _count_r30_users) + def _get_start_of_day(self): + """ + Returns millisecond unixtime for start of UTC day. + """ + now = datetime.datetime.utcnow() + today_start = datetime.datetime(now.year, now.month, + now.day, tzinfo=tz.tzutc()) + return int(time.mktime(today_start.timetuple())) * 1000 + def generate_user_daily_visits(self): """ Generates daily visit data for use in cohort/ retention analysis """ def _generate_user_daily_visits(txn): + logger.info("Calling _generate_user_daily_visits") + today_start = self._get_start_of_day() + a_day_in_milliseconds = 24 * 60 * 60 * 1000 - # determine timestamp of the day start - now = datetime.datetime.utcnow() - today_start = datetime.datetime(now.year, now.month, - now.day, tzinfo=tz.tzutc()) - today_start_time = int(time.mktime(today_start.timetuple())) * 1000 - logger.info(today_start_time) sql = """ INSERT INTO user_daily_visits (user_id, device_id, timestamp) - SELECT user_id, device_id, ? - FROM user_ips AS u - LEFT JOIN user_daily_visits USING (user_id, device_id) - WHERE last_seen > ? AND timestamp IS NULL - GROUP BY user_id, device_id; - """ - txn.execute(sql, (today_start_time, today_start_time)) + SELECT u.user_id, u.device_id, ? + FROM user_ips AS u + LEFT JOIN ( + SELECT user_id, device_id, timestamp FROM user_daily_visits + WHERE timestamp IS ? + ) udv + ON u.user_id = udv.user_id AND u.device_id=udv.device_id + WHERE last_seen > ? AND last_seen <= ? AND udv.timestamp IS NULL + """ + + # This means that the day has rolled over but there could still + # be entries from the previous day. There is an edge case + # where if the user logs in at 23:59 and overwrites their + # last_seen at 00:01 then they will not be counted in the + # previous day's stats - it is important that the query is run + # to minimise this case. + if today_start > self._last_user_visit_update: + yesterday_start = today_start - a_day_in_milliseconds + txn.execute(sql, (yesterday_start, yesterday_start, + self._last_user_visit_update, today_start)) + self._last_user_visit_update = today_start + + txn.execute(sql, (today_start, today_start, + self._last_user_visit_update, + today_start + a_day_in_milliseconds)) + # Update _last_user_visit_update to now. The reason to do this + # rather just clamping to the beginning of the day is to limit + # the size of the join - meaning that the query can be run more + # frequently + + now = datetime.datetime.utcnow() + self._last_user_visit_update = int(time.mktime(now.timetuple())) * 1000 return self.runInteraction("generate_user_daily_visits", _generate_user_daily_visits) From c92a8aa5780a71b5ab0227f204606004b16ff7fb Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Tue, 15 May 2018 17:31:11 +0100 Subject: [PATCH 157/310] pep8 --- synapse/app/homeserver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f25eaf9ffc..41ee8b0a69 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -17,7 +17,6 @@ import gc import logging import os import sys -import datetime import synapse import synapse.config.logger From 31c2502ca8d74797f1eae553690830b8c132aed9 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 16 May 2018 09:46:43 +0100 Subject: [PATCH 158/310] style and further contraining query --- synapse/storage/__init__.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 52f176a03c..6f324e075f 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -368,6 +368,7 @@ class DataStore(RoomMemberStore, RoomStore, logger.info("Calling _generate_user_daily_visits") today_start = self._get_start_of_day() a_day_in_milliseconds = 24 * 60 * 60 * 1000 + now = self.clock.time_msec() sql = """ INSERT INTO user_daily_visits (user_id, device_id, timestamp) @@ -386,23 +387,26 @@ class DataStore(RoomMemberStore, RoomStore, # where if the user logs in at 23:59 and overwrites their # last_seen at 00:01 then they will not be counted in the # previous day's stats - it is important that the query is run - # to minimise this case. + # often to minimise this case. if today_start > self._last_user_visit_update: yesterday_start = today_start - a_day_in_milliseconds - txn.execute(sql, (yesterday_start, yesterday_start, - self._last_user_visit_update, today_start)) + txn.execute(sql, ( + yesterday_start, yesterday_start, + self._last_user_visit_update, today_start + )) self._last_user_visit_update = today_start - txn.execute(sql, (today_start, today_start, - self._last_user_visit_update, - today_start + a_day_in_milliseconds)) + txn.execute(sql, ( + today_start, today_start, + self._last_user_visit_update, + now + )) # Update _last_user_visit_update to now. The reason to do this # rather just clamping to the beginning of the day is to limit # the size of the join - meaning that the query can be run more # frequently - now = datetime.datetime.utcnow() - self._last_user_visit_update = int(time.mktime(now.timetuple())) * 1000 + self._last_user_visit_update = now return self.runInteraction("generate_user_daily_visits", _generate_user_daily_visits) From a2204cc9cc3f9edb9ae6c1f564b8f315e62d1978 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 16 May 2018 09:47:20 +0100 Subject: [PATCH 159/310] remove unused method recurring_user_daily_visit_stats --- synapse/app/homeserver.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 41ee8b0a69..a4648b84d5 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -478,9 +478,6 @@ def run(hs): def generate_user_daily_visit_stats(): hs.get_datastore().generate_user_daily_visits() - def recurring_user_daily_visit_stats(): - clock.looping_call(generate_user_daily_visit_stats, 60 * 60 * 1000) - # Rather than update on per session basis, batch up the requests. # If you increase the loop period, the accuracy of user_daily_visits # table will decrease From be11a02c4f854452687f0f54492bc53b4827637d Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 16 May 2018 10:45:40 +0100 Subject: [PATCH 160/310] remove empty line --- synapse/storage/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 6f324e075f..4551cf8774 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -405,7 +405,6 @@ class DataStore(RoomMemberStore, RoomStore, # rather just clamping to the beginning of the day is to limit # the size of the join - meaning that the query can be run more # frequently - self._last_user_visit_update = now return self.runInteraction("generate_user_daily_visits", From c945af8799671db8fdfc47acf6b286520fb08715 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 16 May 2018 10:52:06 +0100 Subject: [PATCH 161/310] Move and rename variable --- synapse/storage/events.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index f65e18c1ee..22699cb689 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -1825,8 +1825,10 @@ class EventsStore(EventsWorkerStore): ) def _purge_history_txn( - self, txn, room_id, token, delete_local_events, + self, txn, room_id, token_str, delete_local_events, ): + token = RoomStreamToken.parse(token_str) + # Tables that should be pruned: # event_auth # event_backward_extremities @@ -1855,8 +1857,6 @@ class EventsStore(EventsWorkerStore): # furthermore, we might already have the table from a previous (failed) # purge attempt, so let's drop the table first. - token = RoomStreamToken.parse(token) - txn.execute("DROP TABLE IF EXISTS events_to_purge") txn.execute( From 43e6e82c4d368d981f8a0015246d19c333511ace Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 16 May 2018 11:13:31 +0100 Subject: [PATCH 162/310] Comments --- synapse/storage/events.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 22699cb689..93ac80ee48 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -1872,6 +1872,9 @@ class EventsStore(EventsWorkerStore): "CREATE INDEX events_to_purge_should_delete" " ON events_to_purge(should_delete)", ) + + # We do joins against events_to_purge for e.g. calculating state + # groups to purge, etc., so lets make an index. txn.execute( "CREATE INDEX events_to_purge_id" " ON events_to_purge(event_id)", @@ -2119,6 +2122,11 @@ class EventsStore(EventsWorkerStore): # # So, let's stick it at the end so that we don't block event # persistence. + # + # We do this by calculating the minimum depth of the backwards + # extremities. However, the events in event_backward_extremities + # are ones we don't have yet so we need to look at the events that + # point to it via event_edges table. txn.execute(""" SELECT COALESCE(MIN(depth), 0) FROM event_backward_extremities AS eb From 680530cc7f80533b9d63ce9bed04dc28ca87ca13 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 16 May 2018 11:47:26 +0100 Subject: [PATCH 163/310] Clarify comment --- synapse/storage/events.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 93ac80ee48..5ebef98c4f 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -1931,7 +1931,8 @@ class EventsStore(EventsWorkerStore): logger.info("[purge] Finding new backward extremities") # We calculate the new entries for the backward extremeties by finding - # all events that point to events that are to be purged + # events to be purged that are pointed to by events we're not going to + # purge. txn.execute( "SELECT DISTINCT e.event_id FROM events_to_purge AS e" " INNER JOIN event_edges AS ed ON e.event_id = ed.prev_event_id" From 3c099219e0506f81fc333ff9d27cc3af5c365a2c Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 16 May 2018 15:44:59 +0100 Subject: [PATCH 164/310] bump version and changelog for 0.29.0 --- CHANGES.rst | 4 ++++ synapse/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 78d015db9a..b769b0f046 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,7 @@ +Changes in synapse v0.29.0 (2018-05-16) +=========================================== + + Changes in synapse v0.29.0-rc1 (2018-05-14) =========================================== diff --git a/synapse/__init__.py b/synapse/__init__.py index 69c081eb37..d94c20505e 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.29.0-rc1" +__version__ = "0.29.0" From c46367d0d755e810745cae6d0212bdaa12de7645 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 17 May 2018 09:01:09 +0100 Subject: [PATCH 165/310] Move RoomCreationHandler out of synapse.handlers.Handlers Handlers is deprecated nowadays, so let's move this out before I add a new dependency on it. Also fix the docstrings on create_room. --- synapse/handlers/__init__.py | 5 +---- synapse/handlers/room.py | 8 ++++++-- synapse/rest/client/v1/admin.py | 4 ++-- synapse/rest/client/v1/room.py | 5 ++--- synapse/server.py | 5 +++++ synapse/server.pyi | 3 +++ 6 files changed, 19 insertions(+), 11 deletions(-) diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index 8f8fd82eb0..d358842b3e 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -14,9 +14,7 @@ # limitations under the License. from .register import RegistrationHandler -from .room import ( - RoomCreationHandler, RoomContextHandler, -) +from .room import RoomContextHandler from .message import MessageHandler from .federation import FederationHandler from .directory import DirectoryHandler @@ -47,7 +45,6 @@ class Handlers(object): def __init__(self, hs): self.registration_handler = RegistrationHandler(hs) self.message_handler = MessageHandler(hs) - self.room_creation_handler = RoomCreationHandler(hs) self.federation_handler = FederationHandler(hs) self.directory_handler = DirectoryHandler(hs) self.admin_handler = AdminHandler(hs) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 8df8fcbbad..e36426de5a 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -72,10 +72,14 @@ class RoomCreationHandler(BaseHandler): """ Creates a new room. Args: - requester (Requester): The user who requested the room creation. + requester (synapse.types.Requester): + The user who requested the room creation. config (dict) : A dict of configuration options. + ratelimit (bool): set to False to disable the rate limiter Returns: - The new room ID. + Deferred[dict]: + a dict containing the keys `room_id` and, if an alias was + requested, `room_alias`. Raises: SynapseError if the room ID couldn't be stored, or something went horribly wrong. diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index efd5c9873d..646a95c9fb 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -273,8 +273,8 @@ class ShutdownRoomRestServlet(ClientV1RestServlet): def __init__(self, hs): super(ShutdownRoomRestServlet, self).__init__(hs) self.store = hs.get_datastore() - self.handlers = hs.get_handlers() self.state = hs.get_state_handler() + self._room_creation_handler = hs.get_room_creation_handler() self.event_creation_handler = hs.get_event_creation_handler() self.room_member_handler = hs.get_room_member_handler() @@ -296,7 +296,7 @@ class ShutdownRoomRestServlet(ClientV1RestServlet): message = content.get("message", self.DEFAULT_MESSAGE) room_name = content.get("room_name", "Content Violation Notification") - info = yield self.handlers.room_creation_handler.create_room( + info = yield self._room_creation_handler.create_room( room_creator_requester, config={ "preset": "public_chat", diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index fcf9c9ab44..0b984987ed 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -41,7 +41,7 @@ class RoomCreateRestServlet(ClientV1RestServlet): def __init__(self, hs): super(RoomCreateRestServlet, self).__init__(hs) - self.handlers = hs.get_handlers() + self._room_creation_handler = hs.get_room_creation_handler() def register(self, http_server): PATTERNS = "/createRoom" @@ -64,8 +64,7 @@ class RoomCreateRestServlet(ClientV1RestServlet): def on_POST(self, request): requester = yield self.auth.get_user_by_req(request) - handler = self.handlers.room_creation_handler - info = yield handler.create_room( + info = yield self._room_creation_handler.create_room( requester, self.get_room_config(request) ) diff --git a/synapse/server.py b/synapse/server.py index 21cde5b6fc..9e6f3584b2 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -46,6 +46,7 @@ from synapse.handlers.devicemessage import DeviceMessageHandler from synapse.handlers.device import DeviceHandler from synapse.handlers.e2e_keys import E2eKeysHandler from synapse.handlers.presence import PresenceHandler +from synapse.handlers.room import RoomCreationHandler from synapse.handlers.room_list import RoomListHandler from synapse.handlers.room_member import RoomMemberMasterHandler from synapse.handlers.room_member_worker import RoomMemberWorkerHandler @@ -109,6 +110,7 @@ class HomeServer(object): 'federation_server', 'handlers', 'auth', + 'room_creation_handler', 'state_handler', 'state_resolution_handler', 'presence_handler', @@ -227,6 +229,9 @@ class HomeServer(object): def build_simple_http_client(self): return SimpleHttpClient(self) + def build_room_creation_handler(self): + return RoomCreationHandler(self) + def build_state_handler(self): return StateHandler(self) diff --git a/synapse/server.pyi b/synapse/server.pyi index c3a9a3847b..aeda093f27 100644 --- a/synapse/server.pyi +++ b/synapse/server.pyi @@ -40,6 +40,9 @@ class HomeServer(object): def get_deactivate_account_handler(self) -> synapse.handlers.deactivate_account.DeactivateAccountHandler: pass + def get_room_creation_handler(self) -> synapse.handlers.room.RoomCreationHandler: + pass + def get_set_password_handler(self) -> synapse.handlers.set_password.SetPasswordHandler: pass From 616da9eb1d50d9653d2cc122bedecdb0a6ca6487 Mon Sep 17 00:00:00 2001 From: rubo77 Date: Wed, 16 May 2018 23:31:19 +0200 Subject: [PATCH 166/310] postgres.rst: Add instructions how to setup the postgres user and clarify the final step --- docs/postgres.rst | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/docs/postgres.rst b/docs/postgres.rst index 904942ec74..296293e859 100644 --- a/docs/postgres.rst +++ b/docs/postgres.rst @@ -6,7 +6,13 @@ Postgres version 9.4 or later is known to work. Set up database =============== -The PostgreSQL database used *must* have the correct encoding set, otherwise +Assuming your PostgreSQL database user is called ``postgres``, create a user +``synapse_user`` with:: + + su - postgres + createuser --pwprompt synapse_user + +The PostgreSQL database used *must* have the correct encoding set, otherwise it would not be able to store UTF8 strings. To create a database with the correct encoding use, e.g.:: @@ -46,8 +52,8 @@ As with Debian/Ubuntu, postgres support depends on the postgres python connector Synapse config ============== -When you are ready to start using PostgreSQL, add the following line to your -config file:: +When you are ready to start using PostgreSQL, edit the ``database`` section in +your config file to match the following lines:: database: name: psycopg2 @@ -96,9 +102,12 @@ complete, restart synapse. For instance:: cp homeserver.db homeserver.db.snapshot ./synctl start -Assuming your new config file (as described in the section *Synapse config*) -is named ``homeserver-postgres.yaml`` and the SQLite snapshot is at -``homeserver.db.snapshot`` then simply run:: +Copy the old config file into a new config file:: + + cp homeserver.yaml homeserver-postgres.yaml + +Edit the database section as described in the section *Synapse config* above +and with the SQLite snapshot located at ``homeserver.db.snapshot`` simply run:: synapse_port_db --sqlite-database homeserver.db.snapshot \ --postgres-config homeserver-postgres.yaml @@ -117,6 +126,11 @@ run:: --postgres-config homeserver-postgres.yaml Once that has completed, change the synapse config to point at the PostgreSQL -database configuration file ``homeserver-postgres.yaml`` (i.e. rename it to -``homeserver.yaml``) and restart synapse. Synapse should now be running against -PostgreSQL. +database configuration file ``homeserver-postgres.yaml``: + + ./synctl stop + mv homeserver.yaml homeserver-old-sqlite.yaml + mv homeserver-postgres.yaml homeserver.yaml + ./synctl start + +Synapse should now be running against PostgreSQL. From fed62e21ad2c0f622ed47283b41064e9fecaa00b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 17 May 2018 11:34:28 +0100 Subject: [PATCH 167/310] Infrastructure for a server notices room Server Notices use a special room which the user can't dismiss. They are created on demand when some other bit of the code calls send_notice. (This doesn't actually do much yet becuse we don't call send_notice anywhere) --- synapse/config/homeserver.py | 5 +- synapse/config/server_notices_config.py | 85 ++++++++++++++ synapse/handlers/register.py | 14 +++ synapse/handlers/room.py | 16 ++- synapse/handlers/room_member.py | 40 ++++++- synapse/server.py | 5 + synapse/server.pyi | 12 ++ synapse/server_notices/__init__.py | 0 .../server_notices/server_notices_manager.py | 110 ++++++++++++++++++ 9 files changed, 280 insertions(+), 7 deletions(-) create mode 100644 synapse/config/server_notices_config.py create mode 100644 synapse/server_notices/__init__.py create mode 100644 synapse/server_notices/server_notices_manager.py diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index fb6bd3b421..1dea2ad024 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -38,6 +38,7 @@ from .spam_checker import SpamCheckerConfig from .groups import GroupsConfig from .user_directory import UserDirectoryConfig from .consent_config import ConsentConfig +from .server_notices_config import ServerNoticesConfig class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, @@ -47,7 +48,9 @@ class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, JWTConfig, PasswordConfig, EmailConfig, WorkerConfig, PasswordAuthProviderConfig, PushConfig, SpamCheckerConfig, GroupsConfig, UserDirectoryConfig, - ConsentConfig): + ConsentConfig, + ServerNoticesConfig, + ): pass diff --git a/synapse/config/server_notices_config.py b/synapse/config/server_notices_config.py new file mode 100644 index 0000000000..51ddcac112 --- /dev/null +++ b/synapse/config/server_notices_config.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from ._base import Config +from synapse.types import UserID + +DEFAULT_CONFIG = """\ +# Server Notices room configuration +# +# Uncomment this section to enable a room which can be used to send notices +# from the server to users. It is a special room which cannot be left; notices +# come from a special "notices" user id. +# +# If you uncomment this section, you *must* define the system_mxid_localpart +# setting, which defines the id of the user which will be used to send the +# notices. +# +# It's also possible to override the room name, or the display name of the +# "notices" user. +# +# server_notices: +# system_mxid_localpart: notices +# system_mxid_display_name: "Server Notices" +# room_name: "Server Notices" +""" + + +class ServerNoticesConfig(Config): + def __init__(self): + super(ServerNoticesConfig, self).__init__() + + """The MXID to use for server notices. + + None if server notices are not enabled. + + type: str|None + """ + self.server_notices_mxid = None + + """The display name to use for the server notices user. + + None if server notices are not enabled. + + type: str|None + """ + self.server_notices_mxid_display_name = None + + """The name to use for the server notices room. + + None if server notices are not enabled. + + (TODO: i18n) + + type: str|None + """ + self.server_notices_room_name = None + + def read_config(self, config): + c = config.get("server_notices") + if c is None: + return + + mxid_localpart = c['system_mxid_localpart'] + self.server_notices_mxid = UserID( + mxid_localpart, self.server_name, + ).to_string() + self.server_notices_mxid_display_name = c.get( + 'system_mxid_display_name', 'Server Notices', + ) + + self.server_notices_room_name = c.get('room_name', "Server Notices") + + def default_config(self, **kwargs): + return DEFAULT_CONFIG diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index f83c6b3cf8..7e52adda3c 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -34,6 +34,11 @@ logger = logging.getLogger(__name__) class RegistrationHandler(BaseHandler): def __init__(self, hs): + """ + + Args: + hs (synapse.server.HomeServer): + """ super(RegistrationHandler, self).__init__(hs) self.auth = hs.get_auth() @@ -49,6 +54,7 @@ class RegistrationHandler(BaseHandler): self._generate_user_id_linearizer = Linearizer( name="_generate_user_id_linearizer", ) + self._server_notices_mxid = hs.config.server_notices_mxid @defer.inlineCallbacks def check_username(self, localpart, guest_access_token=None, @@ -338,6 +344,14 @@ class RegistrationHandler(BaseHandler): yield identity_handler.bind_threepid(c, user_id) def check_user_id_not_appservice_exclusive(self, user_id, allowed_appservice=None): + # don't allow people to register the server notices mxid + if self._server_notices_mxid is not None: + if user_id == self._server_notices_mxid: + raise SynapseError( + 400, "This user ID is reserved.", + errcode=Codes.EXCLUSIVE + ) + # valid user IDs must not clash with any user ID namespaces claimed by # application services. services = self.store.get_app_services() diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index e36426de5a..5e9fa95a2d 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -68,7 +68,8 @@ class RoomCreationHandler(BaseHandler): self.event_creation_handler = hs.get_event_creation_handler() @defer.inlineCallbacks - def create_room(self, requester, config, ratelimit=True): + def create_room(self, requester, config, ratelimit=True, + creator_join_profile=None): """ Creates a new room. Args: @@ -76,6 +77,14 @@ class RoomCreationHandler(BaseHandler): The user who requested the room creation. config (dict) : A dict of configuration options. ratelimit (bool): set to False to disable the rate limiter + + creator_join_profile (dict|None): + Set to override the displayname and avatar for the creating + user in this room. If unset, displayname and avatar will be + derived from the user's profile. If set, should contain the + values to go in the body of the 'join' event (typically + `avatar_url` and/or `displayname`. + Returns: Deferred[dict]: a dict containing the keys `room_id` and, if an alias was @@ -180,7 +189,8 @@ class RoomCreationHandler(BaseHandler): initial_state=initial_state, creation_content=creation_content, room_alias=room_alias, - power_level_content_override=config.get("power_level_content_override", {}) + power_level_content_override=config.get("power_level_content_override", {}), + creator_join_profile=creator_join_profile, ) if "name" in config: @@ -260,6 +270,7 @@ class RoomCreationHandler(BaseHandler): creation_content, room_alias, power_level_content_override, + creator_join_profile, ): def create(etype, content, **kwargs): e = { @@ -303,6 +314,7 @@ class RoomCreationHandler(BaseHandler): room_id, "join", ratelimit=False, + content=creator_join_profile, ) # We treat the power levels override specially as this needs to be one diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 714583f1d5..9f8975e1f2 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -17,11 +17,14 @@ import abc import logging +from six.moves import http_client + from signedjson.key import decode_verify_key_bytes from signedjson.sign import verify_signed_json from twisted.internet import defer from unpaddedbase64 import decode_base64 +import synapse.server import synapse.types from synapse.api.constants import ( EventTypes, Membership, @@ -46,6 +49,11 @@ class RoomMemberHandler(object): __metaclass__ = abc.ABCMeta def __init__(self, hs): + """ + + Args: + hs (synapse.server.HomeServer): + """ self.hs = hs self.store = hs.get_datastore() self.auth = hs.get_auth() @@ -63,6 +71,7 @@ class RoomMemberHandler(object): self.clock = hs.get_clock() self.spam_checker = hs.get_spam_checker() + self._server_notices_mxid = self.config.server_notices_mxid @abc.abstractmethod def _remote_join(self, requester, remote_room_hosts, room_id, user, content): @@ -289,12 +298,28 @@ class RoomMemberHandler(object): is_blocked = yield self.store.is_room_blocked(room_id) if is_blocked: raise SynapseError(403, "This room has been blocked on this server") + else: + if self._is_server_notice_room(room_id): + # we don't allow people to reject invites to, or leave, the + # server notice room. + raise SynapseError( + http_client.FORBIDDEN, + "You cannot leave this room", + ) - if effective_membership_state == "invite": + if effective_membership_state == Membership.INVITE: block_invite = False - is_requester_admin = yield self.auth.is_server_admin( - requester.user, - ) + + if (self._server_notices_mxid is not None and + requester.user.to_string() == self._server_notices_mxid): + # allow the server notices mxid to send invites + is_requester_admin = True + + else: + is_requester_admin = yield self.auth.is_server_admin( + requester.user, + ) + if not is_requester_admin: if self.config.block_non_admin_invites: logger.info( @@ -844,6 +869,13 @@ class RoomMemberHandler(object): defer.returnValue(False) + @defer.inlineCallbacks + def _is_server_notice_room(self, room_id): + if self._server_notices_mxid is None: + defer.returnValue(False) + user_ids = yield self.store.get_users_in_room(room_id) + defer.returnValue(self._server_notices_mxid in user_ids) + class RoomMemberMasterHandler(RoomMemberHandler): def __init__(self, hs): diff --git a/synapse/server.py b/synapse/server.py index 9e6f3584b2..85f54cd047 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -72,6 +72,7 @@ from synapse.rest.media.v1.media_repository import ( MediaRepository, MediaRepositoryResource, ) +from synapse.server_notices.server_notices_manager import ServerNoticesManager from synapse.state import StateHandler, StateResolutionHandler from synapse.storage import DataStore from synapse.streams.events import EventSources @@ -156,6 +157,7 @@ class HomeServer(object): 'spam_checker', 'room_member_handler', 'federation_registry', + 'server_notices_manager', ] def __init__(self, hostname, **kwargs): @@ -398,6 +400,9 @@ class HomeServer(object): def build_federation_registry(self): return FederationHandlerRegistry() + def build_server_notices_manager(self): + return ServerNoticesManager(self) + def remove_pusher(self, app_id, push_key, user_id): return self.get_pusherpool().remove_pusher(app_id, push_key, user_id) diff --git a/synapse/server.pyi b/synapse/server.pyi index aeda093f27..6fbe15168d 100644 --- a/synapse/server.pyi +++ b/synapse/server.pyi @@ -1,4 +1,5 @@ import synapse.api.auth +import synapse.config.homeserver import synapse.federation.transaction_queue import synapse.federation.transport.client import synapse.handlers @@ -8,11 +9,16 @@ import synapse.handlers.device import synapse.handlers.e2e_keys import synapse.handlers.set_password import synapse.rest.media.v1.media_repository +import synapse.server_notices.server_notices_manager import synapse.state import synapse.storage class HomeServer(object): + @property + def config(self) -> synapse.config.homeserver.HomeServerConfig: + pass + def get_auth(self) -> synapse.api.auth.Auth: pass @@ -43,6 +49,9 @@ class HomeServer(object): def get_room_creation_handler(self) -> synapse.handlers.room.RoomCreationHandler: pass + def get_event_creation_handler(self) -> synapse.handlers.message.EventCreationHandler: + pass + def get_set_password_handler(self) -> synapse.handlers.set_password.SetPasswordHandler: pass @@ -57,3 +66,6 @@ class HomeServer(object): def get_media_repository(self) -> synapse.rest.media.v1.media_repository.MediaRepository: pass + + def get_server_notices_manager(self) -> synapse.server_notices.server_notices_manager.ServerNoticesManager: + pass diff --git a/synapse/server_notices/__init__.py b/synapse/server_notices/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py new file mode 100644 index 0000000000..0d48aeb95e --- /dev/null +++ b/synapse/server_notices/server_notices_manager.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from twisted.internet import defer + +from synapse.api.constants import EventTypes, Membership, RoomCreationPreset +from synapse.types import create_requester +from synapse.util.caches.descriptors import cachedInlineCallbacks + +logger = logging.getLogger(__name__) + + +class ServerNoticesManager(object): + def __init__(self, hs): + """ + + Args: + hs (synapse.server.HomeServer): + """ + + self._store = hs.get_datastore() + self._config = hs.config + self._room_creation_handler = hs.get_room_creation_handler() + self._event_creation_handler = hs.get_event_creation_handler() + + def is_enabled(self): + return self._config.server_notices_mxid is not None + + @defer.inlineCallbacks + def send_notice(self, user_id, event_content): + room_id = yield self.get_notice_room_for_user(user_id) + + system_mxid = self._config.server_notices_mxid + requester = create_requester(system_mxid) + + logger.info("Sending server notice to %s", user_id) + + yield self._event_creation_handler.create_and_send_nonmember_event( + requester, { + "type": EventTypes.Message, + "room_id": room_id, + "sender": system_mxid, + "content": event_content, + }, + ratelimit=False, + ) + + @cachedInlineCallbacks() + def get_notice_room_for_user(self, user_id): + """Get the room for notices for a given user + + If we have not yet created a notice room for this user, create it + + Args: + user_id (str): complete user id for the user we want a room for + + Returns: + str: room id of notice room. + """ + if not self.is_enabled(): + raise Exception("Server notices not enabled") + + rooms = yield self._store.get_rooms_for_user_where_membership_is( + user_id, [Membership.INVITE, Membership.JOIN], + ) + system_mxid = self._config.server_notices_mxid + for room in rooms: + user_ids = yield self._store.get_users_in_room(room.room_id) + if system_mxid in user_ids: + # we found a room which our user shares with the system notice + # user + logger.info("Using room %s", room.room_id) + defer.returnValue(room.room_id) + + # apparently no existing notice room: create a new one + logger.info("Creating server notices room for %s", user_id) + + requester = create_requester(system_mxid) + info = yield self._room_creation_handler.create_room( + requester, + config={ + "preset": RoomCreationPreset.PRIVATE_CHAT, + "name": self._config.server_notices_room_name, + "power_level_content_override": { + "users_default": -10, + }, + "invite": (user_id,) + }, + ratelimit=False, + creator_join_profile={ + "displayname": self._config.server_notices_mxid_display_name, + }, + ) + room_id = info['room_id'] + + logger.info("Created server notices room %s for %s", room_id, user_id) + defer.returnValue(room_id) From 1b1b47aec66e4627b9e365b4a2561cac23aa191c Mon Sep 17 00:00:00 2001 From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com> Date: Wed, 16 May 2018 09:59:45 +0100 Subject: [PATCH 168/310] Reference synapse docker image and docker-compose --- README.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 28fbe45de6..4fe54b0c90 100644 --- a/README.rst +++ b/README.rst @@ -157,8 +157,9 @@ if you prefer. In case of problems, please see the _`Troubleshooting` section below. -Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a Dockerfile to automate the -above in Docker at https://hub.docker.com/r/avhost/docker-matrix/tags/ +There is an offical synapse image available at https://hub.docker.com/r/matrixdotorg/synapse/tags/ which can be used with the docker-compose file available at `contrib/docker`. Further information on this including configuration options is available in `contrib/docker/README.md`. + +Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a Dockerfile to automate a synapse server in a single Docker image, at https://hub.docker.com/r/avhost/docker-matrix/tags/ Also, Martin Giess has created an auto-deployment process with vagrant/ansible, tested with VirtualBox/AWS/DigitalOcean - see https://github.com/EMnify/matrix-synapse-auto-deploy From 82b0361f02b2860da572356c317de9690e88a146 Mon Sep 17 00:00:00 2001 From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com> Date: Wed, 16 May 2018 10:01:03 +0100 Subject: [PATCH 169/310] Document macaroon env var correctly --- contrib/docker/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index aed56646c2..dee5e92cab 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -111,8 +111,6 @@ variables are available for configuration: * ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname. * ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous statistics reporting back to the Matrix project which helps us to get funding. -* ``SYNAPSE_MACAROON_SECRET_KEY`` (mandatory) secret for signing access tokens - to the server, set this to a proper random key. * ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if you run your own TLS-capable reverse proxy). * ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on @@ -132,6 +130,8 @@ Shared secrets, that will be initialized to random values if not set: * ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if registration is disable. +* ``SYNAPSE_MACAROON_SECRET_KEY`` secret for signing access tokens + to the server. Database specific values (will use SQLite if not set): From 450f500d0c96919a657225da63662555531531cb Mon Sep 17 00:00:00 2001 From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com> Date: Wed, 16 May 2018 10:10:31 +0100 Subject: [PATCH 170/310] Note that secrets need to be retained. --- contrib/docker/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index dee5e92cab..ef022f9dc1 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -1,6 +1,6 @@ # Synapse Docker -This Docker image will run Synapse as a single process. It does not provide any +The `matrixdotorg/synapse` Docker image will run Synapse as a single process. It does not provide any database server or TURN server that you should run separately. If you run a Postgres server, you should simply have it in the same Compose @@ -37,10 +37,15 @@ then run the server: docker-compose up -d ``` +If secrets are not specified in the environment variables, they will be generated +as part of the startup. Please ensure these secrets are kept between launches of the +Docker container, as their loss may require users to log in again. + ### Manual configuration A sample ``docker-compose.yml`` is provided, including example labels for -reverse proxying and other artifacts. +reverse proxying and other artifacts. The docker-compose file is an example, +please comment/uncomment sections that are not suitable for your usecase. Specify a ``SYNAPSE_CONFIG_PATH``, preferably to a persistent path, to use manual configuration. To generate a fresh ``homeserver.yaml``, simply run: From a2da6de40ee52fafbb6ab1828c32abdcaf9e1865 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Thu, 17 May 2018 12:44:07 +0100 Subject: [PATCH 171/310] light grammar changes --- contrib/docker/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index ef022f9dc1..61592109cb 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -1,9 +1,9 @@ # Synapse Docker -The `matrixdotorg/synapse` Docker image will run Synapse as a single process. It does not provide any -database server or TURN server that you should run separately. +The `matrixdotorg/synapse` Docker image will run Synapse as a single process. It does not provide a +database server or a TURN server, you should run these separately. -If you run a Postgres server, you should simply have it in the same Compose +If you run a Postgres server, you should simply include it in the same Compose project or set the proper environment variables and the image will automatically use that server. @@ -37,7 +37,7 @@ then run the server: docker-compose up -d ``` -If secrets are not specified in the environment variables, they will be generated +If secrets are not specified in the environment variables, they will be generated as part of the startup. Please ensure these secrets are kept between launches of the Docker container, as their loss may require users to log in again. From d43d480d86e8e5564643ec8d80da4ed4a0174934 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 17 May 2018 18:22:19 +0100 Subject: [PATCH 172/310] Remove unused `update_external_syncs` This method isn't used anywhere. Burninate it. --- synapse/handlers/presence.py | 55 ------------------------------------ 1 file changed, 55 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 585f3e4da2..91218e40e6 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -463,61 +463,6 @@ class PresenceHandler(object): syncing_user_ids.update(user_ids) return syncing_user_ids - @defer.inlineCallbacks - def update_external_syncs(self, process_id, syncing_user_ids): - """Update the syncing users for an external process - - Args: - process_id(str): An identifier for the process the users are - syncing against. This allows synapse to process updates - as user start and stop syncing against a given process. - syncing_user_ids(set(str)): The set of user_ids that are - currently syncing on that server. - """ - - # Grab the previous list of user_ids that were syncing on that process - prev_syncing_user_ids = ( - self.external_process_to_current_syncs.get(process_id, set()) - ) - # Grab the current presence state for both the users that are syncing - # now and the users that were syncing before this update. - prev_states = yield self.current_state_for_users( - syncing_user_ids | prev_syncing_user_ids - ) - updates = [] - time_now_ms = self.clock.time_msec() - - # For each new user that is syncing check if we need to mark them as - # being online. - for new_user_id in syncing_user_ids - prev_syncing_user_ids: - prev_state = prev_states[new_user_id] - if prev_state.state == PresenceState.OFFLINE: - updates.append(prev_state.copy_and_replace( - state=PresenceState.ONLINE, - last_active_ts=time_now_ms, - last_user_sync_ts=time_now_ms, - )) - else: - updates.append(prev_state.copy_and_replace( - last_user_sync_ts=time_now_ms, - )) - - # For each user that is still syncing or stopped syncing update the - # last sync time so that we will correctly apply the grace period when - # they stop syncing. - for old_user_id in prev_syncing_user_ids: - prev_state = prev_states[old_user_id] - updates.append(prev_state.copy_and_replace( - last_user_sync_ts=time_now_ms, - )) - - yield self._update_states(updates) - - # Update the last updated time for the process. We expire the entries - # if we don't receive an update in the given timeframe. - self.external_process_last_updated_ms[process_id] = self.clock.time_msec() - self.external_process_to_current_syncs[process_id] = syncing_user_ids - @defer.inlineCallbacks def update_external_syncs_row(self, process_id, user_id, is_syncing, sync_time_msec): """Update the syncing users for an external process as a delta. From 88d34053326328f3ce62d424c72ed0244f1955d4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 17 May 2018 18:33:45 +0100 Subject: [PATCH 173/310] fix missing yield for server_notices_room --- synapse/handlers/room_member.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 9f8975e1f2..9b99f0f9d7 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -299,9 +299,10 @@ class RoomMemberHandler(object): if is_blocked: raise SynapseError(403, "This room has been blocked on this server") else: - if self._is_server_notice_room(room_id): - # we don't allow people to reject invites to, or leave, the - # server notice room. + # we don't allow people to reject invites to, or leave, the + # server notice room. + is_blocked = yield self._is_server_notice_room(room_id) + if is_blocked: raise SynapseError( http_client.FORBIDDEN, "You cannot leave this room", From 6958459b50a3b29611f98d620d0400d245f8b60f Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Thu, 17 May 2018 21:35:07 +0100 Subject: [PATCH 174/310] bump version, change log --- CHANGES.rst | 8 +++++++- synapse/__init__.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b769b0f046..851cb9b2d7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,12 @@ +Changes in synapse v0.29.1 (2018-05-17) +========================================== +Changes: + +* Update docker documentation (PR #3222) + Changes in synapse v0.29.0 (2018-05-16) =========================================== - +Not changes since v0.29.0-rc1 Changes in synapse v0.29.0-rc1 (2018-05-14) =========================================== diff --git a/synapse/__init__.py b/synapse/__init__.py index d94c20505e..b11a1d3ea3 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.29.0" +__version__ = "0.29.1" From 8b1c856d815c8ab939cd5a272665d80a22ee5d33 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 18 May 2018 09:15:35 +0100 Subject: [PATCH 175/310] Fix error in handling receipts Fixes an error which has been happening ever since #2158 (v0.21.0-rc1): > TypeError: argument of type 'ObservableDeferred' is not iterable fixes #3234 --- synapse/storage/receipts.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index 2f95e7e82a..709c69a926 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -297,18 +297,22 @@ class ReceiptsWorkerStore(SQLBaseStore): if receipt_type != "m.read": return - # Returns an ObservableDeferred + # Returns either an ObservableDeferred or the raw result res = self.get_users_with_read_receipts_in_room.cache.get( room_id, None, update_metrics=False, ) - if res: - if isinstance(res, defer.Deferred) and res.called: + # first handle the Deferred case + if isinstance(res, defer.Deferred): + if res.called: res = res.result - if user_id in res: - # We'd only be adding to the set, so no point invalidating if the - # user is already there - return + else: + res = None + + if res and user_id in res: + # We'd only be adding to the set, so no point invalidating if the + # user is already there + return self.get_users_with_read_receipts_in_room.invalidate((room_id,)) From d10707c8105f4cc7f079ed45960f8333c6101c86 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 18 May 2018 11:00:55 +0100 Subject: [PATCH 176/310] Replace inline docstrings with "Attributes" in class docstring --- synapse/config/server_notices_config.py | 40 ++++++++++--------------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/synapse/config/server_notices_config.py b/synapse/config/server_notices_config.py index 51ddcac112..ccef8d2ec5 100644 --- a/synapse/config/server_notices_config.py +++ b/synapse/config/server_notices_config.py @@ -37,33 +37,25 @@ DEFAULT_CONFIG = """\ class ServerNoticesConfig(Config): + """Configuration for the server notices room. + + Attributes: + server_notices_mxid (str|None): + The MXID to use for server notices. + None if server notices are not enabled. + + server_notices_mxid_display_name (str|None): + The display name to use for the server notices user. + None if server notices are not enabled. + + server_notices_room_name (str|None): + The name to use for the server notices room. + None if server notices are not enabled. + """ def __init__(self): super(ServerNoticesConfig, self).__init__() - - """The MXID to use for server notices. - - None if server notices are not enabled. - - type: str|None - """ self.server_notices_mxid = None - - """The display name to use for the server notices user. - - None if server notices are not enabled. - - type: str|None - """ self.server_notices_mxid_display_name = None - - """The name to use for the server notices room. - - None if server notices are not enabled. - - (TODO: i18n) - - type: str|None - """ self.server_notices_room_name = None def read_config(self, config): @@ -78,7 +70,7 @@ class ServerNoticesConfig(Config): self.server_notices_mxid_display_name = c.get( 'system_mxid_display_name', 'Server Notices', ) - + # todo: i18n self.server_notices_room_name = c.get('room_name', "Server Notices") def default_config(self, **kwargs): From 26305788fe23448d65d0cd0358df547e816574f2 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 18 May 2018 11:18:39 +0100 Subject: [PATCH 177/310] Make sure we reject attempts to invite the notices user --- synapse/handlers/federation.py | 8 ++++++++ synapse/handlers/room_member.py | 7 +++++++ synapse/server_notices/server_notices_manager.py | 5 +++++ 3 files changed, 20 insertions(+) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index f39233d846..ba3ede8024 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -81,6 +81,7 @@ class FederationHandler(BaseHandler): self.pusher_pool = hs.get_pusherpool() self.spam_checker = hs.get_spam_checker() self.event_creation_handler = hs.get_event_creation_handler() + self._server_notices_mxid = hs.config.server_notices_mxid # When joining a room we need to queue any events for that room up self.room_queues = {} @@ -1180,6 +1181,13 @@ class FederationHandler(BaseHandler): if not self.is_mine_id(event.state_key): raise SynapseError(400, "The invite event must be for this server") + # block any attempts to invite the server notices mxid + if event.state_key == self._server_notices_mxid: + raise SynapseError( + http_client.FORBIDDEN, + "Cannot invite this user", + ) + event.internal_metadata.outlier = True event.internal_metadata.invite_from_remote = True diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 9b99f0f9d7..5263f09aaa 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -309,6 +309,13 @@ class RoomMemberHandler(object): ) if effective_membership_state == Membership.INVITE: + # block any attempts to invite the server notices mxid + if target.to_string() == self._server_notices_mxid: + raise SynapseError( + http_client.FORBIDDEN, + "Cannot invite this user", + ) + block_invite = False if (self._server_notices_mxid is not None and diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index 0d48aeb95e..15086600ed 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -78,6 +78,11 @@ class ServerNoticesManager(object): ) system_mxid = self._config.server_notices_mxid for room in rooms: + # it's worth noting that there is an asymmetry here in that we + # expect the user to be invited or joined, but the system user must + # be joined. This is kinda deliberate, in that if somebody somehow + # manages to invite the system user to a room, that doesn't make it + # the server notices room. user_ids = yield self._store.get_users_in_room(room.room_id) if system_mxid in user_ids: # we found a room which our user shares with the system notice From 011e1f4010776cc67eeead09c20e18da35637c00 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 18 May 2018 11:22:12 +0100 Subject: [PATCH 178/310] Better docstrings --- synapse/server_notices/server_notices_manager.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index 15086600ed..f535b9c9da 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -37,10 +37,26 @@ class ServerNoticesManager(object): self._event_creation_handler = hs.get_event_creation_handler() def is_enabled(self): + """Checks if server notices are enabled on this server. + + Returns: + bool + """ return self._config.server_notices_mxid is not None @defer.inlineCallbacks def send_notice(self, user_id, event_content): + """Send a notice to the given user + + Creates the server notices room, if none exists. + + Args: + user_id (str): mxid of user to send event to. + event_content (dict): content of event to send + + Returns: + Deferrred[None] + """ room_id = yield self.get_notice_room_for_user(user_id) system_mxid = self._config.server_notices_mxid From 861f8a9b218eccea95ffb5c7d50fe0380beeae43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20T=C3=B6tterman?= Date: Fri, 18 May 2018 16:20:08 +0300 Subject: [PATCH 179/310] Add lxml to docker image for web previews MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paul Tötterman --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8085f3d354..565341fee3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,12 @@ FROM docker.io/python:2-alpine3.7 -RUN apk add --no-cache --virtual .nacl_deps su-exec build-base libffi-dev zlib-dev libressl-dev libjpeg-turbo-dev linux-headers postgresql-dev +RUN apk add --no-cache --virtual .nacl_deps su-exec build-base libffi-dev zlib-dev libressl-dev libjpeg-turbo-dev linux-headers postgresql-dev libxslt-dev COPY . /synapse # A wheel cache may be provided in ./cache for faster build RUN cd /synapse \ - && pip install --upgrade pip setuptools psycopg2 \ + && pip install --upgrade pip setuptools psycopg2 lxml \ && mkdir -p /synapse/cache \ && pip install -f /synapse/cache --upgrade --process-dependency-links . \ && mv /synapse/contrib/docker/start.py /synapse/contrib/docker/conf / \ From ef466b3a131c0a6c93027a8f8194538678f920c1 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Fri, 18 May 2018 15:51:21 +0100 Subject: [PATCH 180/310] fix psql compatability bug --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4551cf8774..ac264b5d25 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -376,7 +376,7 @@ class DataStore(RoomMemberStore, RoomStore, FROM user_ips AS u LEFT JOIN ( SELECT user_id, device_id, timestamp FROM user_daily_visits - WHERE timestamp IS ? + WHERE timestamp = ? ) udv ON u.user_id = udv.user_id AND u.device_id=udv.device_id WHERE last_seen > ? AND last_seen <= ? AND udv.timestamp IS NULL From 644aac5f73dbacfe4cb47af09e804b9d1d2788a4 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Fri, 18 May 2018 17:10:35 +0100 Subject: [PATCH 181/310] Tighter filtering for user_daily_visits --- synapse/storage/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index ac264b5d25..979fa22438 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -379,7 +379,11 @@ class DataStore(RoomMemberStore, RoomStore, WHERE timestamp = ? ) udv ON u.user_id = udv.user_id AND u.device_id=udv.device_id - WHERE last_seen > ? AND last_seen <= ? AND udv.timestamp IS NULL + INNER JOIN users ON users.name=u.user_id + WHERE last_seen > ? AND last_seen <= ? + AND udv.timestamp IS NULL AND users.is_guest=0 + AND users.appservice_id IS NULL + GROUP BY u.user_id, u.device_id """ # This means that the day has rolled over but there could still From b932b4ea256fcbfa112f006ccb2e1c10ef195c17 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Mon, 30 Apr 2018 14:36:39 +0200 Subject: [PATCH 182/310] use repr, not str Signed-off-by: Adrian Tschira --- synapse/http/site.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/site.py b/synapse/http/site.py index 202a990508..75e32924e9 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -56,7 +56,7 @@ class SynapseRequest(Request): def __repr__(self): # We overwrite this so that we don't log ``access_token`` - return '<%s at 0x%x method=%s uri=%s clientproto=%s site=%s>' % ( + return '<%s at 0x%x method=%r uri=%r clientproto=%r site=%r>' % ( self.__class__.__name__, id(self), self.method, From aafb0f6b0d7db313ac54a8e5e933970feae4bff3 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Mon, 30 Apr 2018 10:44:31 +0200 Subject: [PATCH 183/310] py3-ize url preview --- synapse/rest/media/v1/preview_url_resource.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py index 2839207abc..565cef2b8d 100644 --- a/synapse/rest/media/v1/preview_url_resource.py +++ b/synapse/rest/media/v1/preview_url_resource.py @@ -24,7 +24,9 @@ import shutil import sys import traceback import simplejson as json -import urlparse + +from six.moves import urllib_parse as urlparse +from six import string_types from twisted.web.server import NOT_DONE_YET from twisted.internet import defer @@ -590,8 +592,8 @@ def _iterate_over_text(tree, *tags_to_ignore): # to be returned. elements = iter([tree]) while True: - el = elements.next() - if isinstance(el, basestring): + el = next(elements) + if isinstance(el, string_types): yield el elif el is not None and el.tag not in tags_to_ignore: # el.text is the text before the first child, so we can immediately From 73cbdef5f773ddd091713d71ac9c63639d7c0825 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Tue, 15 May 2018 17:55:46 +0200 Subject: [PATCH 184/310] fix py3 intern and remove unnecessary py3 encode Signed-off-by: Adrian Tschira --- synapse/util/caches/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index 4adae96681..329ccbb866 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -16,6 +16,9 @@ import synapse.metrics import os +from six.moves import intern +import six + CACHE_SIZE_FACTOR = float(os.environ.get("SYNAPSE_CACHE_FACTOR", 0.5)) metrics = synapse.metrics.get_metrics_for("synapse.util.caches") @@ -66,7 +69,9 @@ def intern_string(string): return None try: - string = string.encode("ascii") + if six.PY2: + string = string.encode("ascii") + return intern(string) except UnicodeEncodeError: return string From dcc235b47d694d9ef7181b379d4e38bed2677028 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Mon, 30 Apr 2018 16:38:23 +0200 Subject: [PATCH 185/310] use stand-in value if maxint is not available Signed-off-by: Adrian Tschira --- synapse/storage/_base.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 2262776ab2..962cc270bc 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -30,6 +30,12 @@ import threading logger = logging.getLogger(__name__) +try: + MAX_TXN_ID = sys.maxint - 1 +except AttributeError: + # python 3 does not have a maximum int value + MAX_TXN_ID = 2**63 - 1 + sql_logger = logging.getLogger("synapse.storage.SQL") transaction_logger = logging.getLogger("synapse.storage.txn") perf_logger = logging.getLogger("synapse.storage.TIME") @@ -222,7 +228,7 @@ class SQLBaseStore(object): # We don't really need these to be unique, so lets stop it from # growing really large. - self._TXN_ID = (self._TXN_ID + 1) % (sys.maxint - 1) + self._TXN_ID = (self._TXN_ID + 1) % (MAX_TXN_ID) name = "%s-%x" % (desc, txn_id, ) From 45b55e23d34371986ee0da5be784a1f2134fd58a Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Sun, 29 Apr 2018 13:54:38 +0200 Subject: [PATCH 186/310] Add batch_iter to utils There's a frequent idiom I noticed where an iterable is split up into a number of chunks/batches. Unfortunately that method does not work with iterators like dict.keys() in python3. This implementation works with iterators. Signed-off-by: Adrian Tschira --- synapse/util/__init__.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 814a7bf71b..fc11e26623 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -20,6 +20,8 @@ from twisted.internet import defer, reactor, task import time import logging +from itertools import islice + logger = logging.getLogger(__name__) @@ -79,3 +81,19 @@ class Clock(object): except Exception: if not ignore_errs: raise + + +def batch_iter(iterable, size): + """batch an iterable up into tuples with a maximum size + + Args: + iterable (iterable): the iterable to slice + size (int): the maximum batch size + + Returns: + an iterator over the chunks + """ + # make sure we can deal with iterables like lists too + sourceiter = iter(iterable) + # call islice until it returns an empty tuple + return iter(lambda: tuple(islice(sourceiter, size)), ()) From d9fe2b2d9dbb539da32f969b8bd752159fd5eb6f Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Sun, 15 Apr 2018 21:43:35 +0200 Subject: [PATCH 187/310] Replace some more comparisons with six plus a bonus b"" string I missed last time Signed-off-by: Adrian Tschira --- synapse/events/utils.py | 4 +++- synapse/events/validator.py | 6 ++++-- synapse/groups/groups_server.py | 4 +++- synapse/http/matrixfederationclient.py | 8 +++++--- synapse/push/push_rule_evaluator.py | 4 +++- synapse/rest/client/v1/presence.py | 8 +++++--- 6 files changed, 23 insertions(+), 11 deletions(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 824f4a42e3..29ae086786 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -20,6 +20,8 @@ from frozendict import frozendict import re +from six import string_types + # Split strings on "." but not "\." This uses a negative lookbehind assertion for '\' # (? Date: Sat, 28 Apr 2018 13:19:12 +0200 Subject: [PATCH 188/310] replace some iteritems with six Signed-off-by: Adrian Tschira --- synapse/handlers/device.py | 14 ++++++++------ synapse/handlers/e2e_keys.py | 13 +++++++------ synapse/handlers/groups_local.py | 3 ++- synapse/handlers/presence.py | 15 ++++++++------- synapse/handlers/sync.py | 14 ++++++++------ synapse/handlers/user_directory.py | 3 ++- synapse/metrics/process_collector.py | 3 ++- synapse/push/bulk_push_rule_evaluator.py | 13 +++++++------ synapse/replication/tcp/protocol.py | 9 +++++---- synapse/replication/tcp/resource.py | 3 ++- synapse/rest/media/v1/media_repository.py | 3 ++- synapse/storage/client_ips.py | 6 ++++-- synapse/storage/devices.py | 9 +++++---- synapse/storage/end_to_end_keys.py | 6 ++++-- synapse/storage/event_push_actions.py | 4 +++- 15 files changed, 69 insertions(+), 49 deletions(-) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index f7457a7082..31bd0e60c6 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -26,6 +26,8 @@ from ._base import BaseHandler import logging +from six import itervalues, iteritems + logger = logging.getLogger(__name__) @@ -318,7 +320,7 @@ class DeviceHandler(BaseHandler): # The user may have left the room # TODO: Check if they actually did or if we were just invited. if room_id not in room_ids: - for key, event_id in current_state_ids.iteritems(): + for key, event_id in iteritems(current_state_ids): etype, state_key = key if etype != EventTypes.Member: continue @@ -338,7 +340,7 @@ class DeviceHandler(BaseHandler): # special-case for an empty prev state: include all members # in the changed list if not event_ids: - for key, event_id in current_state_ids.iteritems(): + for key, event_id in iteritems(current_state_ids): etype, state_key = key if etype != EventTypes.Member: continue @@ -354,10 +356,10 @@ class DeviceHandler(BaseHandler): # Check if we've joined the room? If so we just blindly add all the users to # the "possibly changed" users. - for state_dict in prev_state_ids.itervalues(): + for state_dict in itervalues(prev_state_ids): member_event = state_dict.get((EventTypes.Member, user_id), None) if not member_event or member_event != current_member_id: - for key, event_id in current_state_ids.iteritems(): + for key, event_id in iteritems(current_state_ids): etype, state_key = key if etype != EventTypes.Member: continue @@ -367,14 +369,14 @@ class DeviceHandler(BaseHandler): # If there has been any change in membership, include them in the # possibly changed list. We'll check if they are joined below, # and we're not toooo worried about spuriously adding users. - for key, event_id in current_state_ids.iteritems(): + for key, event_id in iteritems(current_state_ids): etype, state_key = key if etype != EventTypes.Member: continue # check if this member has changed since any of the extremities # at the stream_ordering, and add them to the list if so. - for state_dict in prev_state_ids.itervalues(): + for state_dict in itervalues(prev_state_ids): prev_event_id = state_dict.get(key, None) if not prev_event_id or prev_event_id != event_id: if state_key != user_id: diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index 25aec624af..8a2d177539 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -19,6 +19,7 @@ import logging from canonicaljson import encode_canonical_json from twisted.internet import defer +from six import iteritems from synapse.api.errors import ( SynapseError, CodeMessageException, FederationDeniedError, @@ -92,7 +93,7 @@ class E2eKeysHandler(object): remote_queries_not_in_cache = {} if remote_queries: query_list = [] - for user_id, device_ids in remote_queries.iteritems(): + for user_id, device_ids in iteritems(remote_queries): if device_ids: query_list.extend((user_id, device_id) for device_id in device_ids) else: @@ -103,9 +104,9 @@ class E2eKeysHandler(object): query_list ) ) - for user_id, devices in remote_results.iteritems(): + for user_id, devices in iteritems(remote_results): user_devices = results.setdefault(user_id, {}) - for device_id, device in devices.iteritems(): + for device_id, device in iteritems(devices): keys = device.get("keys", None) device_display_name = device.get("device_display_name", None) if keys: @@ -250,9 +251,9 @@ class E2eKeysHandler(object): "Claimed one-time-keys: %s", ",".join(( "%s for %s:%s" % (key_id, user_id, device_id) - for user_id, user_keys in json_result.iteritems() - for device_id, device_keys in user_keys.iteritems() - for key_id, _ in device_keys.iteritems() + for user_id, user_keys in iteritems(json_result) + for device_id, device_keys in iteritems(user_keys) + for key_id, _ in iteritems(device_keys) )), ) diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py index 977993e7d4..dcae083734 100644 --- a/synapse/handlers/groups_local.py +++ b/synapse/handlers/groups_local.py @@ -15,6 +15,7 @@ # limitations under the License. from twisted.internet import defer +from six import iteritems from synapse.api.errors import SynapseError from synapse.types import get_domain_from_id @@ -449,7 +450,7 @@ class GroupsLocalHandler(object): results = {} failed_results = [] - for destination, dest_user_ids in destinations.iteritems(): + for destination, dest_user_ids in iteritems(destinations): try: r = yield self.transport_client.bulk_get_publicised_groups( destination, list(dest_user_ids), diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 91218e40e6..b51f925220 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -25,6 +25,8 @@ The methods that define policy are: from twisted.internet import defer, reactor from contextlib import contextmanager +from six import itervalues, iteritems + from synapse.api.errors import SynapseError from synapse.api.constants import PresenceState from synapse.storage.presence import UserPresenceState @@ -40,7 +42,6 @@ import synapse.metrics import logging - logger = logging.getLogger(__name__) metrics = synapse.metrics.get_metrics_for(__name__) @@ -526,7 +527,7 @@ class PresenceHandler(object): prev_state.copy_and_replace( last_user_sync_ts=time_now_ms, ) - for prev_state in prev_states.itervalues() + for prev_state in itervalues(prev_states) ]) self.external_process_last_updated_ms.pop(process_id, None) @@ -549,14 +550,14 @@ class PresenceHandler(object): for user_id in user_ids } - missing = [user_id for user_id, state in states.iteritems() if not state] + missing = [user_id for user_id, state in iteritems(states) if not state] if missing: # There are things not in our in memory cache. Lets pull them out of # the database. res = yield self.store.get_presence_for_users(missing) states.update(res) - missing = [user_id for user_id, state in states.iteritems() if not state] + missing = [user_id for user_id, state in iteritems(states) if not state] if missing: new = { user_id: UserPresenceState.default(user_id) @@ -1044,7 +1045,7 @@ class PresenceEventSource(object): defer.returnValue((updates.values(), max_token)) else: defer.returnValue(([ - s for s in updates.itervalues() + s for s in itervalues(updates) if s.state != PresenceState.OFFLINE ], max_token)) @@ -1301,11 +1302,11 @@ def get_interested_remotes(store, states, state_handler): # hosts in those rooms. room_ids_to_states, users_to_states = yield get_interested_parties(store, states) - for room_id, states in room_ids_to_states.iteritems(): + for room_id, states in iteritems(room_ids_to_states): hosts = yield state_handler.get_current_hosts_in_room(room_id) hosts_and_states.append((hosts, states)) - for user_id, states in users_to_states.iteritems(): + for user_id, states in iteritems(users_to_states): host = get_domain_from_id(user_id) hosts_and_states.append(([host], states)) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 263e42dded..d0c99c35e3 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -28,6 +28,8 @@ import collections import logging import itertools +from six import itervalues, iteritems + logger = logging.getLogger(__name__) @@ -275,7 +277,7 @@ class SyncHandler(object): # result returned by the event source is poor form (it might cache # the object) room_id = event["room_id"] - event_copy = {k: v for (k, v) in event.iteritems() + event_copy = {k: v for (k, v) in iteritems(event) if k != "room_id"} ephemeral_by_room.setdefault(room_id, []).append(event_copy) @@ -294,7 +296,7 @@ class SyncHandler(object): for event in receipts: room_id = event["room_id"] # exclude room id, as above - event_copy = {k: v for (k, v) in event.iteritems() + event_copy = {k: v for (k, v) in iteritems(event) if k != "room_id"} ephemeral_by_room.setdefault(room_id, []).append(event_copy) @@ -325,7 +327,7 @@ class SyncHandler(object): current_state_ids = frozenset() if any(e.is_state() for e in recents): current_state_ids = yield self.state.get_current_state_ids(room_id) - current_state_ids = frozenset(current_state_ids.itervalues()) + current_state_ids = frozenset(itervalues(current_state_ids)) recents = yield filter_events_for_client( self.store, @@ -382,7 +384,7 @@ class SyncHandler(object): current_state_ids = frozenset() if any(e.is_state() for e in loaded_recents): current_state_ids = yield self.state.get_current_state_ids(room_id) - current_state_ids = frozenset(current_state_ids.itervalues()) + current_state_ids = frozenset(itervalues(current_state_ids)) loaded_recents = yield filter_events_for_client( self.store, @@ -984,7 +986,7 @@ class SyncHandler(object): if since_token: for joined_sync in sync_result_builder.joined: it = itertools.chain( - joined_sync.timeline.events, joined_sync.state.itervalues() + joined_sync.timeline.events, itervalues(joined_sync.state) ) for event in it: if event.type == EventTypes.Member: @@ -1062,7 +1064,7 @@ class SyncHandler(object): newly_left_rooms = [] room_entries = [] invited = [] - for room_id, events in mem_change_events_by_room_id.iteritems(): + for room_id, events in iteritems(mem_change_events_by_room_id): non_joins = [e for e in events if e.membership != Membership.JOIN] has_join = len(non_joins) != len(events) diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 714f0195c8..cd0b7290f0 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -22,6 +22,7 @@ from synapse.util.metrics import Measure from synapse.util.async import sleep from synapse.types import get_localpart_from_id +from six import iteritems logger = logging.getLogger(__name__) @@ -403,7 +404,7 @@ class UserDirectoryHandler(object): if change: users_with_profile = yield self.state.get_current_user_in_room(room_id) - for user_id, profile in users_with_profile.iteritems(): + for user_id, profile in iteritems(users_with_profile): yield self._handle_new_user(room_id, user_id, profile) else: users = yield self.store.get_users_in_public_due_to_room(room_id) diff --git a/synapse/metrics/process_collector.py b/synapse/metrics/process_collector.py index 6fec3de399..50e5b48a2b 100644 --- a/synapse/metrics/process_collector.py +++ b/synapse/metrics/process_collector.py @@ -15,6 +15,7 @@ import os +from six import iteritems TICKS_PER_SEC = 100 BYTES_PER_PAGE = 4096 @@ -55,7 +56,7 @@ def update_resource_metrics(): # line is PID (command) more stats go here ... raw_stats = line.split(") ", 1)[1].split(" ") - for (name, index) in STAT_FIELDS.iteritems(): + for (name, index) in iteritems(STAT_FIELDS): # subtract 3 from the index, because proc(5) is 1-based, and # we've lost the first two fields in PID and COMMAND above stats[name] = int(raw_stats[index - 3]) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 7c680659b6..2f7e77f5f5 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -30,6 +30,7 @@ from synapse.state import POWER_KEY from collections import namedtuple +from six import itervalues, iteritems logger = logging.getLogger(__name__) @@ -126,7 +127,7 @@ class BulkPushRuleEvaluator(object): ) auth_events = yield self.store.get_events(auth_events_ids) auth_events = { - (e.type, e.state_key): e for e in auth_events.itervalues() + (e.type, e.state_key): e for e in itervalues(auth_events) } sender_level = get_user_power_level(event.sender, auth_events) @@ -160,7 +161,7 @@ class BulkPushRuleEvaluator(object): condition_cache = {} - for uid, rules in rules_by_user.iteritems(): + for uid, rules in iteritems(rules_by_user): if event.sender == uid: continue @@ -406,7 +407,7 @@ class RulesForRoom(object): # If the event is a join event then it will be in current state evnts # map but not in the DB, so we have to explicitly insert it. if event.type == EventTypes.Member: - for event_id in member_event_ids.itervalues(): + for event_id in itervalues(member_event_ids): if event_id == event.event_id: members[event_id] = (event.state_key, event.membership) @@ -414,7 +415,7 @@ class RulesForRoom(object): logger.debug("Found members %r: %r", self.room_id, members.values()) interested_in_user_ids = set( - user_id for user_id, membership in members.itervalues() + user_id for user_id, membership in itervalues(members) if membership == Membership.JOIN ) @@ -426,7 +427,7 @@ class RulesForRoom(object): ) user_ids = set( - uid for uid, have_pusher in if_users_with_pushers.iteritems() if have_pusher + uid for uid, have_pusher in iteritems(if_users_with_pushers) if have_pusher ) logger.debug("With pushers: %r", user_ids) @@ -447,7 +448,7 @@ class RulesForRoom(object): ) ret_rules_by_user.update( - item for item in rules_by_user.iteritems() if item[0] is not None + item for item in iteritems(rules_by_user) if item[0] is not None ) self.update_cache(sequence, members, ret_rules_by_user, state_group) diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py index d7d38464b2..7ca1588f6a 100644 --- a/synapse/replication/tcp/protocol.py +++ b/synapse/replication/tcp/protocol.py @@ -68,6 +68,7 @@ import synapse.metrics import struct import fcntl +from six import iterkeys, iteritems metrics = synapse.metrics.get_metrics_for(__name__) @@ -392,7 +393,7 @@ class ServerReplicationStreamProtocol(BaseReplicationStreamProtocol): if stream_name == "ALL": # Subscribe to all streams we're publishing to. - for stream in self.streamer.streams_by_name.iterkeys(): + for stream in iterkeys(self.streamer.streams_by_name): self.subscribe_to_stream(stream, token) else: self.subscribe_to_stream(stream_name, token) @@ -498,7 +499,7 @@ class ClientReplicationStreamProtocol(BaseReplicationStreamProtocol): BaseReplicationStreamProtocol.connectionMade(self) # Once we've connected subscribe to the necessary streams - for stream_name, token in self.handler.get_streams_to_replicate().iteritems(): + for stream_name, token in iteritems(self.handler.get_streams_to_replicate()): self.replicate(stream_name, token) # Tell the server if we have any users currently syncing (should only @@ -633,7 +634,7 @@ metrics.register_callback( lambda: { (k[0], p.name, p.conn_id): count for p in connected_connections - for k, count in p.inbound_commands_counter.counts.iteritems() + for k, count in iteritems(p.inbound_commands_counter.counts) }, labels=["command", "name", "conn_id"], ) @@ -643,7 +644,7 @@ metrics.register_callback( lambda: { (k[0], p.name, p.conn_id): count for p in connected_connections - for k, count in p.outbound_commands_counter.counts.iteritems() + for k, count in iteritems(p.outbound_commands_counter.counts) }, labels=["command", "name", "conn_id"], ) diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index a41af4fd6c..d1c291e17d 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -26,6 +26,7 @@ from synapse.util.metrics import Measure, measure_func import logging import synapse.metrics +from six import itervalues metrics = synapse.metrics.get_metrics_for(__name__) stream_updates_counter = metrics.register_counter( @@ -79,7 +80,7 @@ class ReplicationStreamer(object): # We only support federation stream if federation sending hase been # disabled on the master. self.streams = [ - stream(hs) for stream in STREAMS_MAP.itervalues() + stream(hs) for stream in itervalues(STREAMS_MAP) if stream != FederationStream or not hs.config.send_federation ] diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index 9800ce7581..2ac767d2dc 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -48,6 +48,7 @@ import shutil import cgi import logging from six.moves.urllib import parse as urlparse +from six import iteritems logger = logging.getLogger(__name__) @@ -603,7 +604,7 @@ class MediaRepository(object): thumbnails[(t_width, t_height, r_type)] = r_method # Now we generate the thumbnails for each dimension, store it - for (t_width, t_height, t_type), t_method in thumbnails.iteritems(): + for (t_width, t_height, t_type), t_method in iteritems(thumbnails): # Generate the thumbnail if t_method == "crop": t_byte_source = yield make_deferred_yieldable(threads.deferToThread( diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index ba46907737..ce338514e8 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -22,6 +22,8 @@ from . import background_updates from synapse.util.caches import CACHE_SIZE_FACTOR +from six import iteritems + logger = logging.getLogger(__name__) @@ -99,7 +101,7 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): def _update_client_ips_batch_txn(self, txn, to_update): self.database_engine.lock_table(txn, "user_ips") - for entry in to_update.iteritems(): + for entry in iteritems(to_update): (user_id, access_token, ip), (user_agent, device_id, last_seen) = entry self._simple_upsert_txn( @@ -231,5 +233,5 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): "user_agent": user_agent, "last_seen": last_seen, } - for (access_token, ip), (user_agent, last_seen) in results.iteritems() + for (access_token, ip), (user_agent, last_seen) in iteritems(results) )) diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py index 712106b83a..d149d8392e 100644 --- a/synapse/storage/devices.py +++ b/synapse/storage/devices.py @@ -21,6 +21,7 @@ from synapse.api.errors import StoreError from ._base import SQLBaseStore, Cache from synapse.util.caches.descriptors import cached, cachedList, cachedInlineCallbacks +from six import itervalues, iteritems logger = logging.getLogger(__name__) @@ -360,7 +361,7 @@ class DeviceStore(SQLBaseStore): return (now_stream_id, []) if len(query_map) >= 20: - now_stream_id = max(stream_id for stream_id in query_map.itervalues()) + now_stream_id = max(stream_id for stream_id in itervalues(query_map)) devices = self._get_e2e_device_keys_txn( txn, query_map.keys(), include_all_devices=True @@ -373,13 +374,13 @@ class DeviceStore(SQLBaseStore): """ results = [] - for user_id, user_devices in devices.iteritems(): + for user_id, user_devices in iteritems(devices): # The prev_id for the first row is always the last row before # `from_stream_id` txn.execute(prev_sent_id_sql, (destination, user_id, from_stream_id)) rows = txn.fetchall() prev_id = rows[0][0] - for device_id, device in user_devices.iteritems(): + for device_id, device in iteritems(user_devices): stream_id = query_map[(user_id, device_id)] result = { "user_id": user_id, @@ -483,7 +484,7 @@ class DeviceStore(SQLBaseStore): if devices: user_devices = devices[user_id] results = [] - for device_id, device in user_devices.iteritems(): + for device_id, device in iteritems(user_devices): result = { "device_id": device_id, } diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py index ff8538ddf8..b146487943 100644 --- a/synapse/storage/end_to_end_keys.py +++ b/synapse/storage/end_to_end_keys.py @@ -21,6 +21,8 @@ import simplejson as json from ._base import SQLBaseStore +from six import iteritems + class EndToEndKeyStore(SQLBaseStore): def set_e2e_device_keys(self, user_id, device_id, time_now, device_keys): @@ -81,8 +83,8 @@ class EndToEndKeyStore(SQLBaseStore): query_list, include_all_devices, ) - for user_id, device_keys in results.iteritems(): - for device_id, device_info in device_keys.iteritems(): + for user_id, device_keys in iteritems(results): + for device_id, device_info in iteritems(device_keys): device_info["keys"] = json.loads(device_info.pop("key_json")) defer.returnValue(results) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index f084a5f54b..d0350ee5fe 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -22,6 +22,8 @@ from synapse.util.caches.descriptors import cachedInlineCallbacks import logging import simplejson as json +from six import iteritems + logger = logging.getLogger(__name__) @@ -420,7 +422,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): txn.executemany(sql, ( _gen_entry(user_id, actions) - for user_id, actions in user_id_actions.iteritems() + for user_id, actions in iteritems(user_id_actions) )) return self.runInteraction( From dfa70adc33b8f058acae8a4f4f61a83f9fab47a7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 21 May 2018 16:03:39 +0100 Subject: [PATCH 189/310] Add in flight request metrics This tracks CPU and DB usage while requests are in flight, rather than when we write the response. --- synapse/http/request_metrics.py | 152 +++++++++++++++++++++++++++++++- synapse/http/site.py | 4 +- 2 files changed, 154 insertions(+), 2 deletions(-) diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py index 8c850bf23f..f1e9b41fde 100644 --- a/synapse/http/request_metrics.py +++ b/synapse/http/request_metrics.py @@ -98,14 +98,76 @@ response_size = metrics.register_counter( "response_size", labels=["method", "servlet", "tag"] ) +# In flight metrics are incremented while the requests are in flight, rather +# than when the response was written. + +in_flight_requests_ru_utime = metrics.register_counter( + "in_flight_requests_ru_utime_seconds", labels=["method", "servlet"], +) + +in_flight_requests_ru_stime = metrics.register_counter( + "in_flight_requests_ru_stime_seconds", labels=["method", "servlet"], +) + +in_flight_requests_db_txn_count = metrics.register_counter( + "in_flight_requests_db_txn_count", labels=["method", "servlet"], +) + +# seconds spent waiting for db txns, excluding scheduling time, when processing +# this request +in_flight_requests_db_txn_duration = metrics.register_counter( + "in_flight_requests_db_txn_duration_seconds", labels=["method", "servlet"], +) + +# seconds spent waiting for a db connection, when processing this request +in_flight_requests_db_sched_duration = metrics.register_counter( + "in_flight_requests_db_sched_duration_seconds", labels=["method", "servlet"] +) + +_in_flight_requests_count = metrics.register_gauge( + "in_flight_requests_count", labels=["method", "servlet"] +) + + +# The set of all in flight requests. +_in_flight_requests = set() + + +def _collect_in_flight(): + """Called just before metrics are collected, so we use it to update all + the in flight request metrics + """ + + # Map from (method, name) -> int, the number of in flight requests of that + # type + counts = {} + + for rm in _in_flight_requests: + rm.update_metrics() + key = (rm.method, rm.name,) + counts[key] = counts.get(key, 0) + 1 + + for (method, name), count in counts.iteritems(): + _in_flight_requests_count.set(count, method, name) + + +metrics.register_collector(_collect_in_flight) + class RequestMetrics(object): - def start(self, time_msec, name): + def start(self, time_msec, name, method): self.start = time_msec self.start_context = LoggingContext.current_context() self.name = name + self.method = method + + self._request_stats = _RequestStats.from_context(self.start_context) + + _in_flight_requests.add(self) def stop(self, time_msec, request): + _in_flight_requests.discard(self) + context = LoggingContext.current_context() tag = "" @@ -147,3 +209,91 @@ class RequestMetrics(object): ) response_size.inc_by(request.sentLength, request.method, self.name, tag) + + # We always call this at the end to ensure that we update the metrics + # regardless of whether a call to /metrics while the request was in + # flight. + self.update_metrics() + + def update_metrics(self): + """Updates the in flight metrics with values from this request. + """ + + diff = self._request_stats.update() + + in_flight_requests_ru_utime.inc_by( + diff.ru_utime, self.method, self.name, + ) + + in_flight_requests_ru_stime.inc_by( + diff.ru_stime, self.method, self.name, + ) + + in_flight_requests_db_txn_count.inc_by( + diff.db_txn_count, self.method, self.name, + ) + + in_flight_requests_db_txn_duration.inc_by( + diff.db_txn_duration_ms / 1000., self.method, self.name, + ) + + in_flight_requests_db_sched_duration.inc_by( + diff.db_sched_duration_ms / 1000., self.method, self.name, + ) + + +class _RequestStats(object): + """Keeps tracks of various metrics for an in flight request. + """ + + __slots__ = [ + "context", "ru_utime", "ru_stime", + "db_txn_count", "db_txn_duration_ms", "db_sched_duration_ms", + ] + + def __init__(self, context, ru_utime, ru_stime, db_txn_count, + db_txn_duration_ms, db_sched_duration_ms): + self.context = context + self.ru_utime = ru_utime + self.ru_stime = ru_stime + self.db_txn_count = db_txn_count + self.db_txn_duration_ms = db_txn_duration_ms + self.db_sched_duration_ms = db_sched_duration_ms + + @staticmethod + def from_context(context): + ru_utime, ru_stime = context.get_resource_usage() + + return _RequestStats( + context, + ru_utime, ru_stime, + context.db_txn_count, + context.db_txn_duration_ms, + context.db_sched_duration_ms, + ) + + def update(self): + """Updates the current values and returns the difference between the + old and new values. + + Returns: + _RequestStats: The difference between the old and new values + """ + new = _RequestStats.from_context(self.context) + + diff = _RequestStats( + self.context, + new.ru_utime - self.ru_utime, + new.ru_stime - self.ru_stime, + new.db_txn_count - self.db_txn_count, + new.db_txn_duration_ms - self.db_txn_duration_ms, + new.db_sched_duration_ms - self.db_sched_duration_ms, + ) + + self.ru_utime = new.ru_utime + self.ru_stime = new.ru_stime + self.db_txn_count = new.db_txn_count + self.db_txn_duration_ms = new.db_txn_duration_ms + self.db_sched_duration_ms = new.db_sched_duration_ms + + return diff diff --git a/synapse/http/site.py b/synapse/http/site.py index 202a990508..b608504225 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -85,7 +85,9 @@ class SynapseRequest(Request): def _started_processing(self, servlet_name): self.start_time = int(time.time() * 1000) self.request_metrics = RequestMetrics() - self.request_metrics.start(self.start_time, name=servlet_name) + self.request_metrics.start( + self.start_time, name=servlet_name, method=self.method, + ) self.site.access_logger.info( "%s - %s - Received request: %s %s", From 6d6e7288fe0200353740e03e6218a2781342f3e4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 21 May 2018 16:49:59 +0100 Subject: [PATCH 190/310] Stop the transaction cache caching failures The transaction cache has some code which tries to stop it caching failures, but if the callback function failed straight away, then things would happen backwards and we'd end up with the failure stuck in the cache. --- synapse/rest/client/transactions.py | 22 ++++++----- tests/rest/client/test_transactions.py | 54 ++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 9 deletions(-) diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py index fceca2edeb..93ce0f5348 100644 --- a/synapse/rest/client/transactions.py +++ b/synapse/rest/client/transactions.py @@ -87,19 +87,23 @@ class HttpTransactionCache(object): deferred = fn(*args, **kwargs) - # if the request fails with a Twisted failure, remove it - # from the transaction map. This is done to ensure that we don't - # cache transient errors like rate-limiting errors, etc. + observable = ObservableDeferred(deferred, consumeErrors=False) + self.transactions[txn_key] = (observable, self.clock.time_msec()) + + # if the request fails with an exception, remove it from the + # transaction map. This is done to ensure that we don't cache + # transient errors like rate-limiting errors, etc. + # + # (make sure we add this errback *after* adding the key above, in case + # the deferred has already failed and is running errbacks + # synchronously) def remove_from_map(err): self.transactions.pop(txn_key, None) - return err + # we deliberately do not propagate the error any further, as we + # expect the observers to have reported it. + deferred.addErrback(remove_from_map) - # We don't add any other errbacks to the raw deferred, so we ask - # ObservableDeferred to swallow the error. This is fine as the error will - # still be reported to the observers. - observable = ObservableDeferred(deferred, consumeErrors=True) - self.transactions[txn_key] = (observable, self.clock.time_msec()) return observable.observe() def _cleanup(self): diff --git a/tests/rest/client/test_transactions.py b/tests/rest/client/test_transactions.py index d7cea30260..b650a7772b 100644 --- a/tests/rest/client/test_transactions.py +++ b/tests/rest/client/test_transactions.py @@ -2,6 +2,8 @@ from synapse.rest.client.transactions import HttpTransactionCache from synapse.rest.client.transactions import CLEANUP_PERIOD_MS from twisted.internet import defer from mock import Mock, call + +from synapse.util.logcontext import LoggingContext from tests import unittest from tests.utils import MockClock @@ -39,6 +41,58 @@ class HttpTransactionCacheTestCase(unittest.TestCase): # expect only a single call to do the work cb.assert_called_once_with("some_arg", keyword="arg", changing_args=0) + @defer.inlineCallbacks + def test_does_not_cache_exceptions(self): + """Checks that, if the callback throws an exception, it is called again + for the next request. + """ + called = [False] + + def cb(): + if called[0]: + # return a valid result the second time + return defer.succeed(self.mock_http_response) + + called[0] = True + raise Exception("boo") + + with LoggingContext("test") as test_context: + try: + yield self.cache.fetch_or_execute(self.mock_key, cb) + except Exception as e: + self.assertEqual(e.message, "boo") + self.assertIs(LoggingContext.current_context(), test_context) + + res = yield self.cache.fetch_or_execute(self.mock_key, cb) + self.assertEqual(res, self.mock_http_response) + self.assertIs(LoggingContext.current_context(), test_context) + + @defer.inlineCallbacks + def test_does_not_cache_failures(self): + """Checks that, if the callback returns a failure, it is called again + for the next request. + """ + called = [False] + + def cb(): + if called[0]: + # return a valid result the second time + return defer.succeed(self.mock_http_response) + + called[0] = True + return defer.fail(Exception("boo")) + + with LoggingContext("test") as test_context: + try: + yield self.cache.fetch_or_execute(self.mock_key, cb) + except Exception as e: + self.assertEqual(e.message, "boo") + self.assertIs(LoggingContext.current_context(), test_context) + + res = yield self.cache.fetch_or_execute(self.mock_key, cb) + self.assertEqual(res, self.mock_http_response) + self.assertIs(LoggingContext.current_context(), test_context) + @defer.inlineCallbacks def test_cleans_up(self): cb = Mock( From 6e1cb54a05df3dd614acb022a665458cc1c9698f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 21 May 2018 16:58:20 +0100 Subject: [PATCH 191/310] Fix logcontext leak in HttpTransactionCache ONE DAY I WILL PURGE THE WORLD OF THIS EVIL --- synapse/rest/client/transactions.py | 38 ++++++++++++-------------- tests/rest/client/test_transactions.py | 21 ++++++++++++++ 2 files changed, 38 insertions(+), 21 deletions(-) diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py index 93ce0f5348..20fa6678ef 100644 --- a/synapse/rest/client/transactions.py +++ b/synapse/rest/client/transactions.py @@ -19,6 +19,7 @@ import logging from synapse.api.auth import get_access_token_from_request from synapse.util.async import ObservableDeferred +from synapse.util.logcontext import make_deferred_yieldable, run_in_background logger = logging.getLogger(__name__) @@ -80,31 +81,26 @@ class HttpTransactionCache(object): Returns: Deferred which resolves to a tuple of (response_code, response_dict). """ - try: - return self.transactions[txn_key][0].observe() - except (KeyError, IndexError): - pass # execute the function instead. + if txn_key in self.transactions: + observable = self.transactions[txn_key][0] + else: + # execute the function instead. + deferred = run_in_background(fn, *args, **kwargs) - deferred = fn(*args, **kwargs) + observable = ObservableDeferred(deferred) + self.transactions[txn_key] = (observable, self.clock.time_msec()) - observable = ObservableDeferred(deferred, consumeErrors=False) - self.transactions[txn_key] = (observable, self.clock.time_msec()) + # if the request fails with an exception, remove it + # from the transaction map. This is done to ensure that we don't + # cache transient errors like rate-limiting errors, etc. + def remove_from_map(err): + self.transactions.pop(txn_key, None) + # we deliberately do not propagate the error any further, as we + # expect the observers to have reported it. - # if the request fails with an exception, remove it from the - # transaction map. This is done to ensure that we don't cache - # transient errors like rate-limiting errors, etc. - # - # (make sure we add this errback *after* adding the key above, in case - # the deferred has already failed and is running errbacks - # synchronously) - def remove_from_map(err): - self.transactions.pop(txn_key, None) - # we deliberately do not propagate the error any further, as we - # expect the observers to have reported it. + deferred.addErrback(remove_from_map) - deferred.addErrback(remove_from_map) - - return observable.observe() + return make_deferred_yieldable(observable.observe()) def _cleanup(self): now = self.clock.time_msec() diff --git a/tests/rest/client/test_transactions.py b/tests/rest/client/test_transactions.py index b650a7772b..b5bc2fa255 100644 --- a/tests/rest/client/test_transactions.py +++ b/tests/rest/client/test_transactions.py @@ -3,6 +3,7 @@ from synapse.rest.client.transactions import CLEANUP_PERIOD_MS from twisted.internet import defer from mock import Mock, call +from synapse.util import async from synapse.util.logcontext import LoggingContext from tests import unittest from tests.utils import MockClock @@ -41,6 +42,26 @@ class HttpTransactionCacheTestCase(unittest.TestCase): # expect only a single call to do the work cb.assert_called_once_with("some_arg", keyword="arg", changing_args=0) + @defer.inlineCallbacks + def test_logcontexts_with_async_result(self): + @defer.inlineCallbacks + def cb(): + yield async.sleep(0) + defer.returnValue("yay") + + @defer.inlineCallbacks + def test(): + with LoggingContext("c") as c1: + res = yield self.cache.fetch_or_execute(self.mock_key, cb) + self.assertIs(LoggingContext.current_context(), c1) + self.assertEqual(res, "yay") + + # run the test twice in parallel + d = defer.gatherResults([test(), test()]) + self.assertIs(LoggingContext.current_context(), LoggingContext.sentinel) + yield d + self.assertIs(LoggingContext.current_context(), LoggingContext.sentinel) + @defer.inlineCallbacks def test_does_not_cache_exceptions(self): """Checks that, if the callback throws an exception, it is called again From f258deffcb7c1fbb0389e4e67eaeb14f12c326a7 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 21 May 2018 17:01:15 -0500 Subject: [PATCH 192/310] remove old metrics libs --- synapse/metrics/metric.py | 328 --------------------------- synapse/metrics/process_collector.py | 122 ---------- 2 files changed, 450 deletions(-) delete mode 100644 synapse/metrics/metric.py delete mode 100644 synapse/metrics/process_collector.py diff --git a/synapse/metrics/metric.py b/synapse/metrics/metric.py deleted file mode 100644 index f421e7a93f..0000000000 --- a/synapse/metrics/metric.py +++ /dev/null @@ -1,328 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2015, 2016 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from itertools import chain -import logging -import re - -logger = logging.getLogger(__name__) - - -def flatten(items): - """Flatten a list of lists - - Args: - items: iterable[iterable[X]] - - Returns: - list[X]: flattened list - """ - return list(chain.from_iterable(items)) - - -class BaseMetric(object): - """Base class for metrics which report a single value per label set - """ - - def __init__(self, name, labels=[], alternative_names=[]): - """ - Args: - name (str): principal name for this metric - labels (list(str)): names of the labels which will be reported - for this metric - alternative_names (iterable(str)): list of alternative names for - this metric. This can be useful to provide a migration path - when renaming metrics. - """ - self._names = [name] + list(alternative_names) - self.labels = labels # OK not to clone as we never write it - - def dimension(self): - return len(self.labels) - - def is_scalar(self): - return not len(self.labels) - - def _render_labelvalue(self, value): - return '"%s"' % (_escape_label_value(value),) - - def _render_key(self, values): - if self.is_scalar(): - return "" - return "{%s}" % ( - ",".join(["%s=%s" % (k, self._render_labelvalue(v)) - for k, v in zip(self.labels, values)]) - ) - - def _render_for_labels(self, label_values, value): - """Render this metric for a single set of labels - - Args: - label_values (list[object]): values for each of the labels, - (which get stringified). - value: value of the metric at with these labels - - Returns: - iterable[str]: rendered metric - """ - rendered_labels = self._render_key(label_values) - return ( - "%s%s %.12g" % (name, rendered_labels, value) - for name in self._names - ) - - def render(self): - """Render this metric - - Each metric is rendered as: - - name{label1="val1",label2="val2"} value - - https://prometheus.io/docs/instrumenting/exposition_formats/#text-format-details - - Returns: - iterable[str]: rendered metrics - """ - raise NotImplementedError() - - -class CounterMetric(BaseMetric): - """The simplest kind of metric; one that stores a monotonically-increasing - value that counts events or running totals. - - Example use cases for Counters: - - Number of requests processed - - Number of items that were inserted into a queue - - Total amount of data that a system has processed - Counters can only go up (and be reset when the process restarts). - """ - - def __init__(self, *args, **kwargs): - super(CounterMetric, self).__init__(*args, **kwargs) - - # dict[list[str]]: value for each set of label values. the keys are the - # label values, in the same order as the labels in self.labels. - # - # (if the metric is a scalar, the (single) key is the empty tuple). - self.counts = {} - - # Scalar metrics are never empty - if self.is_scalar(): - self.counts[()] = 0. - - def inc_by(self, incr, *values): - if len(values) != self.dimension(): - raise ValueError( - "Expected as many values to inc() as labels (%d)" % (self.dimension()) - ) - - # TODO: should assert that the tag values are all strings - - if values not in self.counts: - self.counts[values] = incr - else: - self.counts[values] += incr - - def inc(self, *values): - self.inc_by(1, *values) - - def render(self): - return flatten( - self._render_for_labels(k, self.counts[k]) - for k in sorted(self.counts.keys()) - ) - - -class GaugeMetric(BaseMetric): - """A metric that can go up or down - """ - - def __init__(self, *args, **kwargs): - super(GaugeMetric, self).__init__(*args, **kwargs) - - # dict[list[str]]: value for each set of label values. the keys are the - # label values, in the same order as the labels in self.labels. - # - # (if the metric is a scalar, the (single) key is the empty tuple). - self.guages = {} - - def set(self, v, *values): - if len(values) != self.dimension(): - raise ValueError( - "Expected as many values to inc() as labels (%d)" % (self.dimension()) - ) - - # TODO: should assert that the tag values are all strings - - self.guages[values] = v - - def render(self): - return flatten( - self._render_for_labels(k, self.guages[k]) - for k in sorted(self.guages.keys()) - ) - - -class CallbackMetric(BaseMetric): - """A metric that returns the numeric value returned by a callback whenever - it is rendered. Typically this is used to implement gauges that yield the - size or other state of some in-memory object by actively querying it.""" - - def __init__(self, name, callback, labels=[]): - super(CallbackMetric, self).__init__(name, labels=labels) - - self.callback = callback - - def render(self): - try: - value = self.callback() - except Exception: - logger.exception("Failed to render %s", self.name) - return ["# FAILED to render " + self.name] - - if self.is_scalar(): - return list(self._render_for_labels([], value)) - - return flatten( - self._render_for_labels(k, value[k]) - for k in sorted(value.keys()) - ) - - -class DistributionMetric(object): - """A combination of an event counter and an accumulator, which counts - both the number of events and accumulates the total value. Typically this - could be used to keep track of method-running times, or other distributions - of values that occur in discrete occurances. - - TODO(paul): Try to export some heatmap-style stats? - """ - - def __init__(self, name, *args, **kwargs): - self.counts = CounterMetric(name + ":count", **kwargs) - self.totals = CounterMetric(name + ":total", **kwargs) - - def inc_by(self, inc, *values): - self.counts.inc(*values) - self.totals.inc_by(inc, *values) - - def render(self): - return self.counts.render() + self.totals.render() - - -class CacheMetric(object): - __slots__ = ( - "name", "cache_name", "hits", "misses", "evicted_size", "size_callback", - ) - - def __init__(self, name, size_callback, cache_name): - self.name = name - self.cache_name = cache_name - - self.hits = 0 - self.misses = 0 - self.evicted_size = 0 - - self.size_callback = size_callback - - def inc_hits(self): - self.hits += 1 - - def inc_misses(self): - self.misses += 1 - - def inc_evictions(self, size=1): - self.evicted_size += size - - def render(self): - size = self.size_callback() - hits = self.hits - total = self.misses + self.hits - - return [ - """%s:hits{name="%s"} %d""" % (self.name, self.cache_name, hits), - """%s:total{name="%s"} %d""" % (self.name, self.cache_name, total), - """%s:size{name="%s"} %d""" % (self.name, self.cache_name, size), - """%s:evicted_size{name="%s"} %d""" % ( - self.name, self.cache_name, self.evicted_size - ), - ] - - -class MemoryUsageMetric(object): - """Keeps track of the current memory usage, using psutil. - - The class will keep the current min/max/sum/counts of rss over the last - WINDOW_SIZE_SEC, by polling UPDATE_HZ times per second - """ - - UPDATE_HZ = 2 # number of times to get memory per second - WINDOW_SIZE_SEC = 30 # the size of the window in seconds - - def __init__(self, hs, psutil): - clock = hs.get_clock() - self.memory_snapshots = [] - - self.process = psutil.Process() - - clock.looping_call(self._update_curr_values, 1000 / self.UPDATE_HZ) - - def _update_curr_values(self): - max_size = self.UPDATE_HZ * self.WINDOW_SIZE_SEC - self.memory_snapshots.append(self.process.memory_info().rss) - self.memory_snapshots[:] = self.memory_snapshots[-max_size:] - - def render(self): - if not self.memory_snapshots: - return [] - - max_rss = max(self.memory_snapshots) - min_rss = min(self.memory_snapshots) - sum_rss = sum(self.memory_snapshots) - len_rss = len(self.memory_snapshots) - - return [ - "process_psutil_rss:max %d" % max_rss, - "process_psutil_rss:min %d" % min_rss, - "process_psutil_rss:total %d" % sum_rss, - "process_psutil_rss:count %d" % len_rss, - ] - - -def _escape_character(m): - """Replaces a single character with its escape sequence. - - Args: - m (re.MatchObject): A match object whose first group is the single - character to replace - - Returns: - str - """ - c = m.group(1) - if c == "\\": - return "\\\\" - elif c == "\"": - return "\\\"" - elif c == "\n": - return "\\n" - return c - - -def _escape_label_value(value): - """Takes a label value and escapes quotes, newlines and backslashes - """ - return re.sub(r"([\n\"\\])", _escape_character, str(value)) diff --git a/synapse/metrics/process_collector.py b/synapse/metrics/process_collector.py deleted file mode 100644 index 6fec3de399..0000000000 --- a/synapse/metrics/process_collector.py +++ /dev/null @@ -1,122 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2015, 2016 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - - -TICKS_PER_SEC = 100 -BYTES_PER_PAGE = 4096 - -HAVE_PROC_STAT = os.path.exists("/proc/stat") -HAVE_PROC_SELF_STAT = os.path.exists("/proc/self/stat") -HAVE_PROC_SELF_LIMITS = os.path.exists("/proc/self/limits") -HAVE_PROC_SELF_FD = os.path.exists("/proc/self/fd") - -# Field indexes from /proc/self/stat, taken from the proc(5) manpage -STAT_FIELDS = { - "utime": 14, - "stime": 15, - "starttime": 22, - "vsize": 23, - "rss": 24, -} - - -stats = {} - -# In order to report process_start_time_seconds we need to know the -# machine's boot time, because the value in /proc/self/stat is relative to -# this -boot_time = None -if HAVE_PROC_STAT: - with open("/proc/stat") as _procstat: - for line in _procstat: - if line.startswith("btime "): - boot_time = int(line.split()[1]) - - -def update_resource_metrics(): - if HAVE_PROC_SELF_STAT: - global stats - with open("/proc/self/stat") as s: - line = s.read() - # line is PID (command) more stats go here ... - raw_stats = line.split(") ", 1)[1].split(" ") - - for (name, index) in STAT_FIELDS.iteritems(): - # subtract 3 from the index, because proc(5) is 1-based, and - # we've lost the first two fields in PID and COMMAND above - stats[name] = int(raw_stats[index - 3]) - - -def _count_fds(): - # Not every OS will have a /proc/self/fd directory - if not HAVE_PROC_SELF_FD: - return 0 - - return len(os.listdir("/proc/self/fd")) - - -def register_process_collector(process_metrics): - process_metrics.register_collector(update_resource_metrics) - - if HAVE_PROC_SELF_STAT: - process_metrics.register_callback( - "cpu_user_seconds_total", - lambda: float(stats["utime"]) / TICKS_PER_SEC - ) - process_metrics.register_callback( - "cpu_system_seconds_total", - lambda: float(stats["stime"]) / TICKS_PER_SEC - ) - process_metrics.register_callback( - "cpu_seconds_total", - lambda: (float(stats["utime"] + stats["stime"])) / TICKS_PER_SEC - ) - - process_metrics.register_callback( - "virtual_memory_bytes", - lambda: int(stats["vsize"]) - ) - process_metrics.register_callback( - "resident_memory_bytes", - lambda: int(stats["rss"]) * BYTES_PER_PAGE - ) - - process_metrics.register_callback( - "start_time_seconds", - lambda: boot_time + int(stats["starttime"]) / TICKS_PER_SEC - ) - - if HAVE_PROC_SELF_FD: - process_metrics.register_callback( - "open_fds", - lambda: _count_fds() - ) - - if HAVE_PROC_SELF_LIMITS: - def _get_max_fds(): - with open("/proc/self/limits") as limits: - for line in limits: - if not line.startswith("Max open files "): - continue - # Line is Max open files $SOFT $HARD - return int(line.split()[3]) - return None - - process_metrics.register_callback( - "max_fds", - lambda: _get_max_fds() - ) From 02c1d29133a2ce3cdb5d1b23ec5edde49d813da9 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 21 May 2018 17:02:20 -0500 Subject: [PATCH 193/310] look at the Prometheus metrics instead --- synapse/app/homeserver.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 75f40fd5a4..6393804a86 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -34,8 +34,7 @@ from synapse.module_api import ModuleApi from synapse.http.additional_resource import AdditionalResource from synapse.http.server import RootRedirect from synapse.http.site import SynapseSite -from synapse.metrics import register_memory_metrics -from synapse.metrics.resource import METRICS_PREFIX, MetricsResource +from synapse.metrics.resource import METRICS_PREFIX from synapse.python_dependencies import CONDITIONAL_REQUIREMENTS, \ check_requirements from synapse.replication.http import ReplicationRestResource, REPLICATION_PREFIX @@ -221,7 +220,8 @@ class SynapseHomeServer(HomeServer): resources[WEB_CLIENT_PREFIX] = build_resource_for_web_client(self) if name == "metrics" and self.get_config().enable_metrics: - resources[METRICS_PREFIX] = MetricsResource(self) + from prometheus_client.twisted import MetricsResource + resources[METRICS_PREFIX] = MetricsResource() if name == "replication": resources[REPLICATION_PREFIX] = ReplicationRestResource(self) @@ -353,8 +353,6 @@ def setup(config_options): hs.get_datastore().start_doing_background_updates() hs.get_federation_client().start_get_pdu_cache() - register_memory_metrics(hs) - reactor.callWhenRunning(start) return hs From c60e0d5e02789217671482bc209da5ac0fd25baf Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 21 May 2018 17:03:20 -0500 Subject: [PATCH 194/310] don't need the resource portion --- synapse/metrics/resource.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/synapse/metrics/resource.py b/synapse/metrics/resource.py index 870f400600..7996e6ab66 100644 --- a/synapse/metrics/resource.py +++ b/synapse/metrics/resource.py @@ -13,27 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -from twisted.web.resource import Resource - -import synapse.metrics - - METRICS_PREFIX = "/_synapse/metrics" - - -class MetricsResource(Resource): - isLeaf = True - - def __init__(self, hs): - Resource.__init__(self) # Resource is old-style, so no super() - - self.hs = hs - - def render_GET(self, request): - response = synapse.metrics.render_all() - - request.setHeader("Content-Type", "text/plain") - request.setHeader("Content-Length", str(len(response))) - - # Encode as UTF-8 (default) - return response.encode() From df9f72d9e5fe264b86005208e0f096156eb03e4b Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 21 May 2018 19:47:37 -0500 Subject: [PATCH 195/310] replacing portions --- synapse/api/auth.py | 2 +- synapse/federation/federation_client.py | 17 +- synapse/federation/federation_server.py | 16 +- synapse/federation/send_queue.py | 8 +- synapse/federation/transaction_queue.py | 47 ++--- synapse/handlers/appservice.py | 19 +- synapse/handlers/presence.py | 58 +++--- synapse/http/client.py | 20 +-- synapse/http/matrixfederationclient.py | 14 +- synapse/metrics/__init__.py | 196 ++++++++------------- synapse/notifier.py | 23 ++- synapse/push/httppusher.py | 13 +- synapse/push/push_rule_evaluator.py | 2 +- synapse/python_dependencies.py | 1 + synapse/replication/tcp/protocol.py | 88 ++++----- synapse/util/caches/__init__.py | 53 ++++-- synapse/util/caches/descriptors.py | 2 +- synapse/util/caches/dictionary_cache.py | 2 +- synapse/util/caches/expiringcache.py | 4 +- synapse/util/caches/response_cache.py | 11 +- synapse/util/caches/stream_change_cache.py | 2 +- synapse/util/metrics.py | 85 ++------- tests/__init__.py | 3 + 23 files changed, 270 insertions(+), 416 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index f17fda6315..b052cf532b 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -57,7 +57,7 @@ class Auth(object): self.TOKEN_NOT_FOUND_HTTP_STATUS = 401 self.token_cache = LruCache(CACHE_SIZE_FACTOR * 10000) - register_cache("token_cache", self.token_cache) + register_cache("cache", "token_cache", self.token_cache) @defer.inlineCallbacks def check_from_context(self, event, context, do_sig_check=True): diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 6163f7c466..2761ffae07 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -32,20 +32,17 @@ from synapse.federation.federation_base import ( FederationBase, event_from_pdu_json, ) -import synapse.metrics from synapse.util import logcontext, unwrapFirstError from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.logcontext import make_deferred_yieldable, run_in_background from synapse.util.logutils import log_function from synapse.util.retryutils import NotRetryingDestination +from prometheus_client import Counter + logger = logging.getLogger(__name__) - -# synapse.federation.federation_client is a silly name -metrics = synapse.metrics.get_metrics_for("synapse.federation.client") - -sent_queries_counter = metrics.register_counter("sent_queries", labels=["type"]) +sent_queries_counter = Counter("synapse_federation_client_sent_queries", "", ["type"]) PDU_RETRY_TIME_MS = 1 * 60 * 1000 @@ -108,7 +105,7 @@ class FederationClient(FederationBase): a Deferred which will eventually yield a JSON object from the response """ - sent_queries_counter.inc(query_type) + sent_queries_counter.labels(query_type).inc() return self.transport_layer.make_query( destination, query_type, args, retry_on_dns_fail=retry_on_dns_fail, @@ -127,7 +124,7 @@ class FederationClient(FederationBase): a Deferred which will eventually yield a JSON object from the response """ - sent_queries_counter.inc("client_device_keys") + sent_queries_counter.labels("client_device_keys").inc() return self.transport_layer.query_client_keys( destination, content, timeout ) @@ -137,7 +134,7 @@ class FederationClient(FederationBase): """Query the device keys for a list of user ids hosted on a remote server. """ - sent_queries_counter.inc("user_devices") + sent_queries_counter.labels("user_devices").inc() return self.transport_layer.query_user_devices( destination, user_id, timeout ) @@ -154,7 +151,7 @@ class FederationClient(FederationBase): a Deferred which will eventually yield a JSON object from the response """ - sent_queries_counter.inc("client_one_time_keys") + sent_queries_counter.labels("client_one_time_keys").inc() return self.transport_layer.claim_client_keys( destination, content, timeout ) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 247ddc89d5..8211273006 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -27,12 +27,13 @@ from synapse.federation.federation_base import ( from synapse.federation.persistence import TransactionActions from synapse.federation.units import Edu, Transaction -import synapse.metrics from synapse.types import get_domain_from_id from synapse.util import async from synapse.util.caches.response_cache import ResponseCache from synapse.util.logutils import log_function +from prometheus_client import Counter + from six import iteritems # when processing incoming transactions, we try to handle multiple rooms in @@ -41,14 +42,11 @@ TRANSACTION_CONCURRENCY_LIMIT = 10 logger = logging.getLogger(__name__) -# synapse.federation.federation_server is a silly name -metrics = synapse.metrics.get_metrics_for("synapse.federation.server") +received_pdus_counter = Counter("synapse_federation_server_received_pdus", "") -received_pdus_counter = metrics.register_counter("received_pdus") +received_edus_counter = Counter("synapse_federation_server_received_edus", "") -received_edus_counter = metrics.register_counter("received_edus") - -received_queries_counter = metrics.register_counter("received_queries", labels=["type"]) +received_queries_counter = Counter("synapse_federation_server_received_queries", "", ["type"]) class FederationServer(FederationBase): @@ -131,7 +129,7 @@ class FederationServer(FederationBase): logger.debug("[%s] Transaction is new", transaction.transaction_id) - received_pdus_counter.inc_by(len(transaction.pdus)) + received_pdus_counter.inc(len(transaction.pdus)) pdus_by_room = {} @@ -292,7 +290,7 @@ class FederationServer(FederationBase): @defer.inlineCallbacks def on_query_request(self, query_type, args): - received_queries_counter.inc(query_type) + received_queries_counter.labels(query_type).inc() resp = yield self.registry.on_query(query_type, args) defer.returnValue((200, resp)) diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py index 0f0c687b37..e6e1888f3a 100644 --- a/synapse/federation/send_queue.py +++ b/synapse/federation/send_queue.py @@ -33,7 +33,7 @@ from .units import Edu from synapse.storage.presence import UserPresenceState from synapse.util.metrics import Measure -import synapse.metrics +from synapse.metrics import LaterGauge from blist import sorteddict from collections import namedtuple @@ -45,9 +45,6 @@ from six import itervalues, iteritems logger = logging.getLogger(__name__) -metrics = synapse.metrics.get_metrics_for(__name__) - - class FederationRemoteSendQueue(object): """A drop in replacement for TransactionQueue""" @@ -77,8 +74,7 @@ class FederationRemoteSendQueue(object): # lambda binds to the queue rather than to the name of the queue which # changes. ARGH. def register(name, queue): - metrics.register_callback( - queue_name + "_size", + LaterGauge("synapse_federation_send_queue_%s_size" % (queue_name,), "", lambda: len(queue), ) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index ded2b1871a..778924a13c 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -26,23 +26,18 @@ from synapse.util.retryutils import NotRetryingDestination, get_retry_limiter from synapse.util.metrics import measure_func from synapse.handlers.presence import format_user_presence_state, get_interested_remotes import synapse.metrics +from synapse.metrics import LaterGauge +from synapse.metrics import ( + sent_edus_counter, sent_transactions_counter, events_processed_counter) + +from prometheus_client import Counter import logging logger = logging.getLogger(__name__) -metrics = synapse.metrics.get_metrics_for(__name__) - -client_metrics = synapse.metrics.get_metrics_for("synapse.federation.client") -sent_pdus_destination_dist = client_metrics.register_distribution( - "sent_pdu_destinations" -) -sent_edus_counter = client_metrics.register_counter("sent_edus") - -sent_transactions_counter = client_metrics.register_counter("sent_transactions") - -events_processed_counter = client_metrics.register_counter("events_processed") +sent_pdus_destination_dist = Counter("synapse_federation_client_sent_pdu_destinations", "") class TransactionQueue(object): @@ -69,8 +64,7 @@ class TransactionQueue(object): # done self.pending_transactions = {} - metrics.register_callback( - "pending_destinations", + LaterGauge("pending_destinations", "", [], lambda: len(self.pending_transactions), ) @@ -94,12 +88,12 @@ class TransactionQueue(object): # Map of destination -> (edu_type, key) -> Edu self.pending_edus_keyed_by_dest = edus_keyed = {} - metrics.register_callback( - "pending_pdus", + LaterGauge( + "pending_pdus", "", [], lambda: sum(map(len, pdus.values())), ) - metrics.register_callback( - "pending_edus", + LaterGauge( + "pending_edus", "", [], lambda: ( sum(map(len, edus.values())) + sum(map(len, presence.values())) @@ -241,18 +235,15 @@ class TransactionQueue(object): now = self.clock.time_msec() ts = yield self.store.get_received_ts(events[-1].event_id) - synapse.metrics.event_processing_lag.set( - now - ts, "federation_sender", - ) - synapse.metrics.event_processing_last_ts.set( - ts, "federation_sender", - ) + synapse.metrics.event_processing_lag.labels( + "federation_sender").set(now - ts) + synapse.metrics.event_processing_last_ts.labels( + "federation_sender").set(ts) - events_processed_counter.inc_by(len(events)) + events_processed_counter.inc(len(events)) - synapse.metrics.event_processing_positions.set( - next_token, "federation_sender", - ) + synapse.metrics.event_processing_positions.labels( + "federation_sender").set(next_token) finally: self._is_processing = False @@ -275,7 +266,7 @@ class TransactionQueue(object): if not destinations: return - sent_pdus_destination_dist.inc_by(len(destinations)) + sent_pdus_destination_dist.inc(len(destinations)) for destination in destinations: self.pending_pdus_by_dest.setdefault(destination, []).append( diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index b596f098fd..a7345331af 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -21,14 +21,13 @@ from synapse.util.metrics import Measure from synapse.util.logcontext import ( make_deferred_yieldable, run_in_background, ) +from prometheus_client import Counter import logging logger = logging.getLogger(__name__) -metrics = synapse.metrics.get_metrics_for(__name__) - -events_processed_counter = metrics.register_counter("events_processed") +events_processed_counter = Counter("synapse_handlers_appservice_events_processed", "") def log_failure(failure): @@ -128,18 +127,12 @@ class ApplicationServicesHandler(object): now = self.clock.time_msec() ts = yield self.store.get_received_ts(events[-1].event_id) - synapse.metrics.event_processing_positions.set( - upper_bound, "appservice_sender", - ) + synapse.metrics.event_processing_positions.labels("appservice_sender").set(upper_bound) - events_processed_counter.inc_by(len(events)) + events_processed_counter.inc(len(events)) - synapse.metrics.event_processing_lag.set( - now - ts, "appservice_sender", - ) - synapse.metrics.event_processing_last_ts.set( - ts, "appservice_sender", - ) + synapse.metrics.event_processing_lag.labels("appservice_sender").set(now - ts) + synapse.metrics.event_processing_last_ts.labels("appservice_sender").set(ts) finally: self.is_processing = False diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 585f3e4da2..06d937ef3a 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -36,27 +36,27 @@ from synapse.util.logutils import log_function from synapse.util.metrics import Measure from synapse.util.wheel_timer import WheelTimer from synapse.types import UserID, get_domain_from_id -import synapse.metrics +from synapse.metrics import LaterGauge import logging +from prometheus_client import Counter + logger = logging.getLogger(__name__) -metrics = synapse.metrics.get_metrics_for(__name__) -notified_presence_counter = metrics.register_counter("notified_presence") -federation_presence_out_counter = metrics.register_counter("federation_presence_out") -presence_updates_counter = metrics.register_counter("presence_updates") -timers_fired_counter = metrics.register_counter("timers_fired") -federation_presence_counter = metrics.register_counter("federation_presence") -bump_active_time_counter = metrics.register_counter("bump_active_time") +notified_presence_counter = Counter("synapse_handler_presence_notified_presence", "") +federation_presence_out_counter = Counter("synapse_handler_presence_federation_presence_out", "") +presence_updates_counter = Counter("synapse_handler_presence_presence_updates", "") +timers_fired_counter = Counter("synapse_handler_presence_timers_fired", "") +federation_presence_counter = Counter("synapse_handler_presence_federation_presence", "") +bump_active_time_counter = Counter("synapse_handler_presence_bump_active_time", "") -get_updates_counter = metrics.register_counter("get_updates", labels=["type"]) +get_updates_counter = Counter("synapse_handler_presence_get_updates", "", ["type"]) -notify_reason_counter = metrics.register_counter("notify_reason", labels=["reason"]) -state_transition_counter = metrics.register_counter( - "state_transition", labels=["from", "to"] +notify_reason_counter = Counter("synapse_handler_presence_notify_reason", "", ["reason"]) +state_transition_counter = Counter("synapse_handler_presence_state_transition", "", ["from", "to"] ) @@ -137,9 +137,9 @@ class PresenceHandler(object): for state in active_presence } - metrics.register_callback( - "user_to_current_state_size", lambda: len(self.user_to_current_state) - ) + LaterGauge( + "user_to_current_state_size", "", [], lambda: len(self.user_to_current_state) + ).register() now = self.clock.time_msec() for state in active_presence: @@ -208,7 +208,7 @@ class PresenceHandler(object): 60 * 1000, ) - metrics.register_callback("wheel_timer_size", lambda: len(self.wheel_timer)) + LaterGauge("wheel_timer_size", "", [], lambda: len(self.wheel_timer)).register() @defer.inlineCallbacks def _on_shutdown(self): @@ -311,10 +311,10 @@ class PresenceHandler(object): # TODO: We should probably ensure there are no races hereafter - presence_updates_counter.inc_by(len(new_states)) + presence_updates_counter.inc(len(new_states)) if to_notify: - notified_presence_counter.inc_by(len(to_notify)) + notified_presence_counter.inc(len(to_notify)) yield self._persist_and_notify(to_notify.values()) self.unpersisted_users_changes |= set(s.user_id for s in new_states) @@ -325,7 +325,7 @@ class PresenceHandler(object): if user_id not in to_notify } if to_federation_ping: - federation_presence_out_counter.inc_by(len(to_federation_ping)) + federation_presence_out_counter.inc(len(to_federation_ping)) self._push_to_remotes(to_federation_ping.values()) @@ -363,7 +363,7 @@ class PresenceHandler(object): for user_id in users_to_check ] - timers_fired_counter.inc_by(len(states)) + timers_fired_counter.inc(len(states)) changes = handle_timeouts( states, @@ -707,7 +707,7 @@ class PresenceHandler(object): updates.append(prev_state.copy_and_replace(**new_fields)) if updates: - federation_presence_counter.inc_by(len(updates)) + federation_presence_counter.inc(len(updates)) yield self._update_states(updates) @defer.inlineCallbacks @@ -982,28 +982,28 @@ def should_notify(old_state, new_state): return False if old_state.status_msg != new_state.status_msg: - notify_reason_counter.inc("status_msg_change") + notify_reason_counter.labels("status_msg_change").inc() return True if old_state.state != new_state.state: - notify_reason_counter.inc("state_change") - state_transition_counter.inc(old_state.state, new_state.state) + notify_reason_counter.labels("state_change").inc() + state_transition_counter.labels(old_state.state, new_state.state).inc() return True if old_state.state == PresenceState.ONLINE: if new_state.currently_active != old_state.currently_active: - notify_reason_counter.inc("current_active_change") + notify_reason_counter.labels("current_active_change").inc() return True if new_state.last_active_ts - old_state.last_active_ts > LAST_ACTIVE_GRANULARITY: # Only notify about last active bumps if we're not currently acive if not new_state.currently_active: - notify_reason_counter.inc("last_active_change_online") + notify_reason_counter.labels("last_active_change_online").inc() return True elif new_state.last_active_ts - old_state.last_active_ts > LAST_ACTIVE_GRANULARITY: # Always notify for a transition where last active gets bumped. - notify_reason_counter.inc("last_active_change_not_online") + notify_reason_counter.labels("last_active_change_not_online").inc() return True return False @@ -1077,14 +1077,14 @@ class PresenceEventSource(object): if changed is not None and len(changed) < 500: # For small deltas, its quicker to get all changes and then # work out if we share a room or they're in our presence list - get_updates_counter.inc("stream") + get_updates_counter.labels("stream").inc() for other_user_id in changed: if other_user_id in users_interested_in: user_ids_changed.add(other_user_id) else: # Too many possible updates. Find all users we can see and check # if any of them have changed. - get_updates_counter.inc("full") + get_updates_counter.labels("full").inc() if from_key: user_ids_changed = stream_change_cache.get_entities_changed( diff --git a/synapse/http/client.py b/synapse/http/client.py index 70a19d9b74..61a1d2e2b3 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -23,7 +23,6 @@ from synapse.http import cancelled_to_request_timed_out_error from synapse.util.async import add_timeout_to_deferred from synapse.util.caches import CACHE_SIZE_FACTOR from synapse.util.logcontext import make_deferred_yieldable -import synapse.metrics from synapse.http.endpoint import SpiderEndpoint from canonicaljson import encode_canonical_json @@ -42,6 +41,7 @@ from twisted.web._newclient import ResponseDone from six import StringIO +from prometheus_client import Counter import simplejson as json import logging import urllib @@ -49,16 +49,8 @@ import urllib logger = logging.getLogger(__name__) -metrics = synapse.metrics.get_metrics_for(__name__) - -outgoing_requests_counter = metrics.register_counter( - "requests", - labels=["method"], -) -incoming_responses_counter = metrics.register_counter( - "responses", - labels=["method", "code"], -) +outgoing_requests_counter = Counter("synapse_http_client_requests", "", ["method"]) +incoming_responses_counter = Counter("synapse_http_client_responses", "", ["method", "code"]) class SimpleHttpClient(object): @@ -95,7 +87,7 @@ class SimpleHttpClient(object): def request(self, method, uri, *args, **kwargs): # A small wrapper around self.agent.request() so we can easily attach # counters to it - outgoing_requests_counter.inc(method) + outgoing_requests_counter.labels(method).inc() logger.info("Sending request %s %s", method, uri) @@ -109,14 +101,14 @@ class SimpleHttpClient(object): ) response = yield make_deferred_yieldable(request_deferred) - incoming_responses_counter.inc(method, response.code) + incoming_responses_counter.labels(method, response.code).inc() logger.info( "Received response to %s %s: %s", method, uri, response.code ) defer.returnValue(response) except Exception as e: - incoming_responses_counter.inc(method, "ERR") + incoming_responses_counter.labels(method, "ERR").inc() logger.info( "Error sending request to %s %s: %s %s", method, uri, type(e).__name__, e.message diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 4b2b85464d..259d3884e2 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -43,19 +43,13 @@ import sys import urllib from six.moves.urllib import parse as urlparse +from prometheus_client import Counter + logger = logging.getLogger(__name__) outbound_logger = logging.getLogger("synapse.http.outbound") -metrics = synapse.metrics.get_metrics_for(__name__) - -outgoing_requests_counter = metrics.register_counter( - "requests", - labels=["method"], -) -incoming_responses_counter = metrics.register_counter( - "responses", - labels=["method", "code"], -) +outgoing_requests_counter = Counter("synapse_http_matrixfederationclient_requests", "", ["method"]) +incoming_responses_counter = Counter("synapse_http_matrixfederationclient_responses", "", ["method", "code"]) MAX_LONG_RETRIES = 10 diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index e3b831db67..973ba6506f 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -18,15 +18,14 @@ import functools import time import gc import platform +import attr + +from prometheus_client import Gauge, Histogram, Counter +from prometheus_client.core import ( + GaugeMetricFamily, CounterMetricFamily, REGISTRY) from twisted.internet import reactor -from .metric import ( - CounterMetric, CallbackMetric, DistributionMetric, CacheMetric, - MemoryUsageMetric, GaugeMetric, -) -from .process_collector import register_process_collector - logger = logging.getLogger(__name__) @@ -34,149 +33,94 @@ logger = logging.getLogger(__name__) running_on_pypy = platform.python_implementation() == 'PyPy' all_metrics = [] all_collectors = [] +all_gauges = {} +@attr.s(hash=True) +class LaterGauge(object): -class Metrics(object): - """ A single Metrics object gives a (mutable) slice view of the all_metrics - dict, allowing callers to easily register new metrics that are namespaced - nicely.""" + name = attr.ib() + desc = attr.ib() + labels = attr.ib(hash=False) + caller = attr.ib() - def __init__(self, name): - self.name_prefix = name + def collect(self): - def make_subspace(self, name): - return Metrics("%s_%s" % (self.name_prefix, name)) + g = GaugeMetricFamily(self.name, self.desc, self.labels) - def register_collector(self, func): - all_collectors.append(func) - - def _register(self, metric_class, name, *args, **kwargs): - full_name = "%s_%s" % (self.name_prefix, name) - - metric = metric_class(full_name, *args, **kwargs) - - all_metrics.append(metric) - return metric - - def register_counter(self, *args, **kwargs): - """ - Returns: - CounterMetric - """ - return self._register(CounterMetric, *args, **kwargs) - - def register_gauge(self, *args, **kwargs): - """ - Returns: - GaugeMetric - """ - return self._register(GaugeMetric, *args, **kwargs) - - def register_callback(self, *args, **kwargs): - """ - Returns: - CallbackMetric - """ - return self._register(CallbackMetric, *args, **kwargs) - - def register_distribution(self, *args, **kwargs): - """ - Returns: - DistributionMetric - """ - return self._register(DistributionMetric, *args, **kwargs) - - def register_cache(self, *args, **kwargs): - """ - Returns: - CacheMetric - """ - return self._register(CacheMetric, *args, **kwargs) - - -def register_memory_metrics(hs): - try: - import psutil - process = psutil.Process() - process.memory_info().rss - except (ImportError, AttributeError): - logger.warn( - "psutil is not installed or incorrect version." - " Disabling memory metrics." - ) - return - metric = MemoryUsageMetric(hs, psutil) - all_metrics.append(metric) - - -def get_metrics_for(pkg_name): - """ Returns a Metrics instance for conveniently creating metrics - namespaced with the given name prefix. """ - - # Convert a "package.name" to "package_name" because Prometheus doesn't - # let us use . in metric names - return Metrics(pkg_name.replace(".", "_")) - - -def render_all(): - strs = [] - - for collector in all_collectors: - collector() - - for metric in all_metrics: try: - strs += metric.render() - except Exception: - strs += ["# FAILED to render"] - logger.exception("Failed to render metric") + calls = self.caller() + except Exception as e: + print(e) + logger.err() + yield g - strs.append("") # to generate a final CRLF + if isinstance(calls, dict): + for k, v in calls.items(): + g.add_metric(k, v) + else: + g.add_metric([], calls) - return "\n".join(strs) + yield g + + def register(self): + if self.name in all_gauges.keys(): + REGISTRY.unregister(all_gauges.pop(self.name)) + + REGISTRY.register(self) + all_gauges[self.name] = self -register_process_collector(get_metrics_for("process")) +# +# Python GC metrics +# +gc_unreachable = Gauge("python_gc_unreachable_total", "Unreachable GC objects", ["gen"]) +gc_time = Histogram("python_gc_time", "Time taken to GC (ms)", ["gen"], buckets=[1, 2, 5, 10, 25, 50, 100, 250, 500, 1000]) -python_metrics = get_metrics_for("python") +class GCCounts(object): + def collect(self): + gc_counts = gc.get_count() -gc_time = python_metrics.register_distribution("gc_time", labels=["gen"]) -gc_unreachable = python_metrics.register_counter("gc_unreachable_total", labels=["gen"]) -python_metrics.register_callback( - "gc_counts", lambda: {(i,): v for i, v in enumerate(gc.get_count())}, labels=["gen"] -) + cm = GaugeMetricFamily("python_gc_counts", "GC cycle counts", labels=["gen"]) + for n, m in enumerate(gc.get_count()): + cm.add_metric([str(n)], m) -reactor_metrics = get_metrics_for("python.twisted.reactor") -tick_time = reactor_metrics.register_distribution("tick_time") -pending_calls_metric = reactor_metrics.register_distribution("pending_calls") + yield cm -synapse_metrics = get_metrics_for("synapse") +REGISTRY.register(GCCounts()) + +# +# Twisted reactor metrics +# + +tick_time = Histogram("python_twisted_reactor_tick_time", "Tick time of the Twisted reactor (ms)", buckets=[1, 2, 5, 10, 50, 100, 250, 500, 1000, 2000]) +pending_calls_metric = Histogram("python_twisted_reactor_pending_calls", "Pending calls", buckets=[1, 2, 5, 10, 25, 50, 100, 250, 500, 1000]) + +# +# Federation Metrics +# + +sent_edus_counter = Counter("synapse_federation_client_sent_edus", "") + +sent_transactions_counter = Counter("synapse_federation_client_sent_transactions", "") + +events_processed_counter = Counter("synapse_federation_client_events_processed", "") # Used to track where various components have processed in the event stream, # e.g. federation sending, appservice sending, etc. -event_processing_positions = synapse_metrics.register_gauge( - "event_processing_positions", labels=["name"], -) +event_processing_positions = Gauge("synapse_event_processing_positions", "", ["name"]) # Used to track the current max events stream position -event_persisted_position = synapse_metrics.register_gauge( - "event_persisted_position", -) +event_persisted_position = Gauge("synapse_event_persisted_position", "") # Used to track the received_ts of the last event processed by various # components -event_processing_last_ts = synapse_metrics.register_gauge( - "event_processing_last_ts", labels=["name"], -) +event_processing_last_ts = Gauge("synapse_event_processing_last_ts", "", ["name"]) # Used to track the lag processing events. This is the time difference # between the last processed event's received_ts and the time it was # finished being processed. -event_processing_lag = synapse_metrics.register_gauge( - "event_processing_lag", labels=["name"], -) - +event_processing_lag = Gauge("synapse_event_processing_lag", "", ["name"]) def runUntilCurrentTimer(func): @@ -206,8 +150,8 @@ def runUntilCurrentTimer(func): # since about 25% of time is actually spent running things triggered by # I/O events, but that is harder to capture without rewriting half the # reactor. - tick_time.inc_by(end - start) - pending_calls_metric.inc_by(num_pending) + tick_time.observe(end - start) + pending_calls_metric.observe(num_pending) if running_on_pypy: return ret @@ -224,8 +168,8 @@ def runUntilCurrentTimer(func): unreachable = gc.collect(i) end = time.time() * 1000 - gc_time.inc_by(end - start, i) - gc_unreachable.inc_by(unreachable, i) + gc_time.labels(i).observe(end - start) + gc_unreachable.labels(i).set(unreachable) return ret diff --git a/synapse/notifier.py b/synapse/notifier.py index 8355c7d621..123e6f1840 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -28,22 +28,19 @@ from synapse.util.logcontext import PreserveLoggingContext, run_in_background from synapse.util.metrics import Measure from synapse.types import StreamToken from synapse.visibility import filter_events_for_client -import synapse.metrics +from synapse.metrics import LaterGauge from collections import namedtuple +from prometheus_client import Counter import logging logger = logging.getLogger(__name__) -metrics = synapse.metrics.get_metrics_for(__name__) +notified_events_counter = Counter("synapse_notifier_notified_events", "") -notified_events_counter = metrics.register_counter("notified_events") - -users_woken_by_stream_counter = metrics.register_counter( - "users_woken_by_stream", labels=["stream"] -) +users_woken_by_stream_counter = Counter("synapse_notifier_users_woken_by_stream", "", ["stream"]) # TODO(paul): Should be shared somewhere @@ -108,7 +105,7 @@ class _NotifierUserStream(object): self.last_notified_ms = time_now_ms noify_deferred = self.notify_deferred - users_woken_by_stream_counter.inc(stream_key) + users_woken_by_stream_counter.labels(stream_key).inc() with PreserveLoggingContext(): self.notify_deferred = ObservableDeferred(defer.Deferred()) @@ -197,14 +194,14 @@ class Notifier(object): all_user_streams.add(x) return sum(stream.count_listeners() for stream in all_user_streams) - metrics.register_callback("listeners", count_listeners) + LaterGauge("listeners", "", [], count_listeners) - metrics.register_callback( - "rooms", + LaterGauge( + "rooms", "", [], lambda: count(bool, self.room_to_user_streams.values()), ) - metrics.register_callback( - "users", + LaterGauge( + "users", "", [], lambda: len(self.user_to_user_stream), ) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index b077e1a446..e22088ad6f 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -20,22 +20,17 @@ from twisted.internet.error import AlreadyCalled, AlreadyCancelled from . import push_rule_evaluator from . import push_tools -import synapse from synapse.push import PusherConfigException from synapse.util.logcontext import LoggingContext from synapse.util.metrics import Measure +from prometheus_client import Counter + logger = logging.getLogger(__name__) -metrics = synapse.metrics.get_metrics_for(__name__) +http_push_processed_counter = Counter("http_pushes_processed", "") -http_push_processed_counter = metrics.register_counter( - "http_pushes_processed", -) - -http_push_failed_counter = metrics.register_counter( - "http_pushes_failed", -) +http_push_failed_counter = Counter("http_pushes_failed", "") class HttpPusher(object): diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index 3601f2d365..c3e6c5c258 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -150,7 +150,7 @@ class PushRuleEvaluatorForEvent(object): # Caches (glob, word_boundary) -> regex for push. See _glob_matches regex_cache = LruCache(50000 * CACHE_SIZE_FACTOR) -register_cache("regex_push_cache", regex_cache) +register_cache("cache", "regex_push_cache", regex_cache) def _glob_matches(glob, value, word_boundary=False): diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 216db4d164..478c497722 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -56,6 +56,7 @@ REQUIREMENTS = { "msgpack-python>=0.3.0": ["msgpack"], "phonenumbers>=8.2.0": ["phonenumbers"], "six": ["six"], + "prometheus_client": ["prometheus_client"], } CONDITIONAL_REQUIREMENTS = { "web_client": { diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py index d7d38464b2..5848f57c5e 100644 --- a/synapse/replication/tcp/protocol.py +++ b/synapse/replication/tcp/protocol.py @@ -60,19 +60,19 @@ from .commands import ( ) from .streams import STREAMS_MAP +from synapse.metrics import LaterGauge from synapse.util.stringutils import random_string -from synapse.metrics.metric import CounterMetric + +from prometheus_client import Counter + +from collections import defaultdict import logging -import synapse.metrics import struct import fcntl - -metrics = synapse.metrics.get_metrics_for(__name__) - -connection_close_counter = metrics.register_counter( - "close_reason", labels=["reason_type"], +connection_close_counter = Counter( + "synapse_replication_tcp_protocol_close_reason", "", ["reason_type"], ) @@ -136,12 +136,8 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver): # The LoopingCall for sending pings. self._send_ping_loop = None - self.inbound_commands_counter = CounterMetric( - "inbound_commands", labels=["command"], - ) - self.outbound_commands_counter = CounterMetric( - "outbound_commands", labels=["command"], - ) + self.inbound_commands_counter = defaultdict(int) + self.outbound_commands_counter = defaultdict(int) def connectionMade(self): logger.info("[%s] Connection established", self.id()) @@ -201,7 +197,7 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver): self.last_received_command = self.clock.time_msec() - self.inbound_commands_counter.inc(cmd_name) + self.inbound_commands_counter[cmd_name] = self.inbound_commands_counter[cmd_name] + 1 cmd_cls = COMMAND_MAP[cmd_name] try: @@ -251,8 +247,7 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver): self._queue_command(cmd) return - self.outbound_commands_counter.inc(cmd.NAME) - + self.outbound_commands_counter[cmd.NAME] = self.outbound_commands_counter[cmd.NAME] + 1 string = "%s %s" % (cmd.NAME, cmd.to_line(),) if "\n" in string: raise Exception("Unexpected newline in command: %r", string) @@ -317,9 +312,9 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver): def connectionLost(self, reason): logger.info("[%s] Replication connection closed: %r", self.id(), reason) if isinstance(reason, Failure): - connection_close_counter.inc(reason.type.__name__) + connection_close_counter.labels(reason.type.__name__).inc() else: - connection_close_counter.inc(reason.__class__.__name__) + connection_close_counter.labels(reason.__class__.__name__).inc() try: # Remove us from list of connections to be monitored @@ -566,14 +561,12 @@ class ClientReplicationStreamProtocol(BaseReplicationStreamProtocol): # The following simply registers metrics for the replication connections -metrics.register_callback( - "pending_commands", +pending_commands = LaterGauge( + "pending_commands", "", ["name", "conn_id"], lambda: { (p.name, p.conn_id): len(p.pending_commands) for p in connected_connections - }, - labels=["name", "conn_id"], -) + }) def transport_buffer_size(protocol): @@ -583,14 +576,12 @@ def transport_buffer_size(protocol): return 0 -metrics.register_callback( - "transport_send_buffer", +transport_send_buffer = LaterGauge( + "synapse_replication_tcp_transport_send_buffer", "", ["name", "conn_id"], lambda: { (p.name, p.conn_id): transport_buffer_size(p) for p in connected_connections - }, - labels=["name", "conn_id"], -) + }) def transport_kernel_read_buffer_size(protocol, read=True): @@ -608,48 +599,37 @@ def transport_kernel_read_buffer_size(protocol, read=True): return 0 -metrics.register_callback( - "transport_kernel_send_buffer", +tcp_transport_kernel_send_buffer = LaterGauge( + "synapse_replication_tcp_transport_kernel_send_buffer", "", ["name", "conn_id"], lambda: { (p.name, p.conn_id): transport_kernel_read_buffer_size(p, False) for p in connected_connections - }, - labels=["name", "conn_id"], -) + }) -metrics.register_callback( - "transport_kernel_read_buffer", +tcp_transport_kernel_read_buffer = LaterGauge( + "synapse_replication_tcp_transport_kernel_read_buffer", "", ["name", "conn_id"], lambda: { (p.name, p.conn_id): transport_kernel_read_buffer_size(p, True) for p in connected_connections - }, - labels=["name", "conn_id"], -) + }) -metrics.register_callback( - "inbound_commands", +tcp_inbound_commands = LaterGauge( + "synapse_replication_tcp_inbound_commands", "", ["command", "name", "conn_id"], lambda: { (k[0], p.name, p.conn_id): count for p in connected_connections - for k, count in p.inbound_commands_counter.counts.iteritems() - }, - labels=["command", "name", "conn_id"], -) + for k, count in p.inbound_commands_counter.items() + }) -metrics.register_callback( - "outbound_commands", +tcp_outbound_commands = LaterGauge( + "synapse_replication_tcp_outbound_commands", "", ["command", "name", "conn_id"], lambda: { (k[0], p.name, p.conn_id): count for p in connected_connections - for k, count in p.outbound_commands_counter.counts.iteritems() - }, - labels=["command", "name", "conn_id"], -) + for k, count in p.outbound_commands_counter.items() + }) # number of updates received for each RDATA stream -inbound_rdata_count = metrics.register_counter( - "inbound_rdata_count", - labels=["stream_name"], -) +inbound_rdata_count = Counter("synapse_replication_tcp_inbound_rdata_count", "", ["stream_name"]) diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index 4adae96681..438dcddf55 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -13,29 +13,52 @@ # See the License for the specific language governing permissions and # limitations under the License. -import synapse.metrics +from prometheus_client.core import GaugeMetricFamily, REGISTRY + import os CACHE_SIZE_FACTOR = float(os.environ.get("SYNAPSE_CACHE_FACTOR", 0.5)) -metrics = synapse.metrics.get_metrics_for("synapse.util.caches") - caches_by_name = {} -# cache_counter = metrics.register_cache( -# "cache", -# lambda: {(name,): len(caches_by_name[name]) for name in caches_by_name.keys()}, -# labels=["name"], -# ) +collectors_by_name = {} +def register_cache(name, cache_name, cache): -def register_cache(name, cache): - caches_by_name[name] = cache - return metrics.register_cache( - "cache", - lambda: len(cache), - name, - ) + # Check if the metric is already registered. Unregister it, if so. + metric_name = "synapse_util_caches_%s:%s" % (name, cache_name,) + if metric_name in collectors_by_name.keys(): + REGISTRY.unregister(collectors_by_name[metric_name]) + class CacheMetric(object): + + hits = 0 + misses = 0 + evicted_size = 0 + + def inc_hits(self): + self.hits += 1 + + def inc_misses(self): + self.misses += 1 + + def inc_evictions(self, size=1): + self.evicted_size += size + + def collect(self): + cache_size = len(cache) + + gm = GaugeMetricFamily(metric_name, "", labels=["size", "hits", "misses", "total"]) + gm.add_metric(["size"], cache_size) + gm.add_metric(["hits"], self.hits) + gm.add_metric(["misses"], self.misses) + gm.add_metric(["total"], self.hits + self.misses) + yield gm + + metric = CacheMetric() + REGISTRY.register(metric) + caches_by_name[cache_name] = cache + collectors_by_name[metric_name] = metric + return metric KNOWN_KEYS = { key: key for key in diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py index 68285a7594..a4188eb099 100644 --- a/synapse/util/caches/descriptors.py +++ b/synapse/util/caches/descriptors.py @@ -80,7 +80,7 @@ class Cache(object): self.name = name self.keylen = keylen self.thread = None - self.metrics = register_cache(name, self.cache) + self.metrics = register_cache("descriptor", name, self.cache) def _on_evicted(self, evicted_count): self.metrics.inc_evictions(evicted_count) diff --git a/synapse/util/caches/dictionary_cache.py b/synapse/util/caches/dictionary_cache.py index 1709e8b429..bdc21e348f 100644 --- a/synapse/util/caches/dictionary_cache.py +++ b/synapse/util/caches/dictionary_cache.py @@ -55,7 +55,7 @@ class DictionaryCache(object): __slots__ = [] self.sentinel = Sentinel() - self.metrics = register_cache(name, self.cache) + self.metrics = register_cache("dictionary", name, self.cache) def check_thread(self): expected_thread = self.thread diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py index 0aa103eecb..ff04c91955 100644 --- a/synapse/util/caches/expiringcache.py +++ b/synapse/util/caches/expiringcache.py @@ -52,12 +52,12 @@ class ExpiringCache(object): self._cache = OrderedDict() - self.metrics = register_cache(cache_name, self) - self.iterable = iterable self._size_estimate = 0 + self.metrics = register_cache("expiring", cache_name, self) + def start(self): if not self._expiry_ms: # Don't bother starting the loop if things never expire diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py index 7f79333e96..a8491b42d5 100644 --- a/synapse/util/caches/response_cache.py +++ b/synapse/util/caches/response_cache.py @@ -17,7 +17,7 @@ import logging from twisted.internet import defer from synapse.util.async import ObservableDeferred -from synapse.util.caches import metrics as cache_metrics +from synapse.util.caches import register_cache from synapse.util.logcontext import make_deferred_yieldable, run_in_background logger = logging.getLogger(__name__) @@ -38,15 +38,16 @@ class ResponseCache(object): self.timeout_sec = timeout_ms / 1000. self._name = name - self._metrics = cache_metrics.register_cache( - "response_cache", - size_callback=lambda: self.size(), - cache_name=name, + self._metrics = register_cache( + "response_cache", name, self ) def size(self): return len(self.pending_result_cache) + def __len__(self): + return self.size() + def get(self, key): """Look up the given key. diff --git a/synapse/util/caches/stream_change_cache.py b/synapse/util/caches/stream_change_cache.py index 941d873ab8..a7fe0397fa 100644 --- a/synapse/util/caches/stream_change_cache.py +++ b/synapse/util/caches/stream_change_cache.py @@ -38,7 +38,7 @@ class StreamChangeCache(object): self._cache = sorteddict() self._earliest_known_stream_pos = current_stream_pos self.name = name - self.metrics = register_cache(self.name, self._cache) + self.metrics = register_cache("cache", self.name, self._cache) for entity, stream_pos in prefilled_cache.items(): self.entity_has_changed(entity, stream_pos) diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index e4b5687a4b..a964286d85 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -15,8 +15,8 @@ from twisted.internet import defer +from prometheus_client import Counter from synapse.util.logcontext import LoggingContext -import synapse.metrics from functools import wraps import logging @@ -24,66 +24,21 @@ import logging logger = logging.getLogger(__name__) +block_counter = Counter("synapse_util_metrics_block_count", "", ["block_name"]) -metrics = synapse.metrics.get_metrics_for(__name__) +block_timer = Counter("synapse_util_metrics_block_time_seconds", "", ["block_name"]) -# total number of times we have hit this block -block_counter = metrics.register_counter( - "block_count", - labels=["block_name"], - alternative_names=( - # the following are all deprecated aliases for the same metric - metrics.name_prefix + x for x in ( - "_block_timer:count", - "_block_ru_utime:count", - "_block_ru_stime:count", - "_block_db_txn_count:count", - "_block_db_txn_duration:count", - ) - ) -) +block_ru_utime = Counter("synapse_util_metrics_block_ru_utime_seconds", "", ["block_name"]) -block_timer = metrics.register_counter( - "block_time_seconds", - labels=["block_name"], - alternative_names=( - metrics.name_prefix + "_block_timer:total", - ), -) +block_ru_stime = Counter("synapse_util_metrics_block_ru_stime_seconds", "", ["block_name"]) -block_ru_utime = metrics.register_counter( - "block_ru_utime_seconds", labels=["block_name"], - alternative_names=( - metrics.name_prefix + "_block_ru_utime:total", - ), -) - -block_ru_stime = metrics.register_counter( - "block_ru_stime_seconds", labels=["block_name"], - alternative_names=( - metrics.name_prefix + "_block_ru_stime:total", - ), -) - -block_db_txn_count = metrics.register_counter( - "block_db_txn_count", labels=["block_name"], - alternative_names=( - metrics.name_prefix + "_block_db_txn_count:total", - ), -) +block_db_txn_count = Counter("synapse_util_metrics_block_db_txn_count", "", ["block_name"]) # seconds spent waiting for db txns, excluding scheduling time, in this block -block_db_txn_duration = metrics.register_counter( - "block_db_txn_duration_seconds", labels=["block_name"], - alternative_names=( - metrics.name_prefix + "_block_db_txn_duration:total", - ), -) +block_db_txn_duration = Counter("synapse_util_metrics_block_db_txn_duration_seconds", "", ["block_name"]) # seconds spent waiting for a db connection, in this block -block_db_sched_duration = metrics.register_counter( - "block_db_sched_duration_seconds", labels=["block_name"], -) +block_db_sched_duration = Counter("synapse_util_metrics_block_db_sched_duration_seconds", "", ["block_name"]) def measure_func(name): @@ -132,8 +87,8 @@ class Measure(object): duration = self.clock.time_msec() - self.start - block_counter.inc(self.name) - block_timer.inc_by(duration, self.name) + block_counter.labels(self.name).inc() + block_timer.labels(self.name).inc(duration) context = LoggingContext.current_context() @@ -150,19 +105,13 @@ class Measure(object): ru_utime, ru_stime = context.get_resource_usage() - block_ru_utime.inc_by(ru_utime - self.ru_utime, self.name) - block_ru_stime.inc_by(ru_stime - self.ru_stime, self.name) - block_db_txn_count.inc_by( - context.db_txn_count - self.db_txn_count, self.name - ) - block_db_txn_duration.inc_by( - (context.db_txn_duration_ms - self.db_txn_duration_ms) / 1000., - self.name - ) - block_db_sched_duration.inc_by( - (context.db_sched_duration_ms - self.db_sched_duration_ms) / 1000., - self.name - ) + block_ru_utime.labels(self.name).inc(ru_utime - self.ru_utime) + block_ru_stime.labels(self.name).inc(ru_stime - self.ru_stime) + block_db_txn_count.labels(self.name).inc(context.db_txn_count - self.db_txn_count) + block_db_txn_duration.labels(self.name).inc( + (context.db_txn_duration_ms - self.db_txn_duration_ms) / 1000.) + block_db_sched_duration.labels(self.name).inc( + (context.db_sched_duration_ms - self.db_sched_duration_ms) / 1000.) if self.created_context: self.start_context.__exit__(exc_type, exc_val, exc_tb) diff --git a/tests/__init__.py b/tests/__init__.py index bfebb0f644..aab20e8e02 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -12,3 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from twisted.trial import util +util.DEFAULT_TIMEOUT_DURATION = 10 From fcc525b0b705703fead3d703c0df62a264ff8ce8 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 21 May 2018 19:48:57 -0500 Subject: [PATCH 196/310] rest of the changes --- synapse/http/request_metrics.py | 104 +++--------- synapse/http/server.py | 4 +- synapse/push/bulk_push_rule_evaluator.py | 28 ++-- synapse/replication/tcp/resource.py | 30 ++-- synapse/storage/_base.py | 17 +- synapse/storage/events.py | 28 ++-- tests/metrics/__init__.py | 0 tests/metrics/test_metric.py | 192 ----------------------- 8 files changed, 67 insertions(+), 336 deletions(-) delete mode 100644 tests/metrics/__init__.py delete mode 100644 tests/metrics/test_metric.py diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py index 8c850bf23f..34a730d5bc 100644 --- a/synapse/http/request_metrics.py +++ b/synapse/http/request_metrics.py @@ -16,86 +16,38 @@ import logging -import synapse.metrics +from prometheus_client.core import Counter, Histogram + from synapse.util.logcontext import LoggingContext logger = logging.getLogger(__name__) -metrics = synapse.metrics.get_metrics_for("synapse.http.server") # total number of responses served, split by method/servlet/tag -response_count = metrics.register_counter( - "response_count", - labels=["method", "servlet", "tag"], - alternative_names=( - # the following are all deprecated aliases for the same metric - metrics.name_prefix + x for x in ( - "_requests", - "_response_time:count", - "_response_ru_utime:count", - "_response_ru_stime:count", - "_response_db_txn_count:count", - "_response_db_txn_duration:count", - ) - ) -) +response_count = Counter("synapse_http_server_response_count", "", ["method", "servlet", "tag"]) -requests_counter = metrics.register_counter( - "requests_received", - labels=["method", "servlet", ], -) +requests_counter = Counter("synapse_http_server_requests_received", "", ["method", "servlet"]) -outgoing_responses_counter = metrics.register_counter( - "responses", - labels=["method", "code"], -) +outgoing_responses_counter = Counter("synapse_http_server_responses", "", ["method", "code"]) -response_timer = metrics.register_counter( - "response_time_seconds", - labels=["method", "servlet", "tag"], - alternative_names=( - metrics.name_prefix + "_response_time:total", - ), -) +response_timer = Histogram("synapse_http_server_response_time_seconds", "", ["method", "servlet", "tag"]) -response_ru_utime = metrics.register_counter( - "response_ru_utime_seconds", labels=["method", "servlet", "tag"], - alternative_names=( - metrics.name_prefix + "_response_ru_utime:total", - ), -) +response_ru_utime = Counter("synapse_http_server_response_ru_utime_seconds", "", ["method", "servlet", "tag"]) -response_ru_stime = metrics.register_counter( - "response_ru_stime_seconds", labels=["method", "servlet", "tag"], - alternative_names=( - metrics.name_prefix + "_response_ru_stime:total", - ), -) +response_ru_stime = Counter("synapse_http_server_response_ru_stime_seconds", "", ["method", "servlet", "tag"]) -response_db_txn_count = metrics.register_counter( - "response_db_txn_count", labels=["method", "servlet", "tag"], - alternative_names=( - metrics.name_prefix + "_response_db_txn_count:total", - ), -) +response_db_txn_count = Counter("synapse_http_server_response_db_txn_count", "", ["method", "servlet", "tag"]) # seconds spent waiting for db txns, excluding scheduling time, when processing # this request -response_db_txn_duration = metrics.register_counter( - "response_db_txn_duration_seconds", labels=["method", "servlet", "tag"], - alternative_names=( - metrics.name_prefix + "_response_db_txn_duration:total", - ), -) +response_db_txn_duration = Counter("synapse_http_server_response_db_txn_duration_seconds", "", ["method", "servlet", "tag"]) # seconds spent waiting for a db connection, when processing this request -response_db_sched_duration = metrics.register_counter( - "response_db_sched_duration_seconds", labels=["method", "servlet", "tag"] +response_db_sched_duration = Counter("synapse_http_request_response_db_sched_duration_seconds", "", ["method", "servlet", "tag"] ) # size in bytes of the response written -response_size = metrics.register_counter( - "response_size", labels=["method", "servlet", "tag"] +response_size = Counter("synapse_http_request_response_size", "", ["method", "servlet", "tag"] ) @@ -119,31 +71,19 @@ class RequestMetrics(object): ) return - outgoing_responses_counter.inc(request.method, str(request.code)) + outgoing_responses_counter.labels(request.method, str(request.code)).inc() - response_count.inc(request.method, self.name, tag) + response_count.labels(request.method, self.name, tag).inc() - response_timer.inc_by( - time_msec - self.start, request.method, - self.name, tag - ) + response_timer.labels(request.method, self.name, tag).observe(time_msec - self.start) ru_utime, ru_stime = context.get_resource_usage() - response_ru_utime.inc_by( - ru_utime, request.method, self.name, tag - ) - response_ru_stime.inc_by( - ru_stime, request.method, self.name, tag - ) - response_db_txn_count.inc_by( - context.db_txn_count, request.method, self.name, tag - ) - response_db_txn_duration.inc_by( - context.db_txn_duration_ms / 1000., request.method, self.name, tag - ) - response_db_sched_duration.inc_by( - context.db_sched_duration_ms / 1000., request.method, self.name, tag - ) + response_ru_utime.labels(request.method, self.name, tag).inc(ru_utime) + response_ru_stime.labels(request.method, self.name, tag).inc(ru_stime) + response_db_txn_count.labels(request.method, self.name, tag).inc(context.db_txn_count) + response_db_txn_duration.labels(request.method, self.name, tag).inc(context.db_txn_duration_ms / 1000.) + response_db_sched_duration.labels(request.method, self.name, tag).inc( + context.db_sched_duration_ms / 1000.) - response_size.inc_by(request.sentLength, request.method, self.name, tag) + response_size.labels(request.method, self.name, tag).inc(request.sentLength) diff --git a/synapse/http/server.py b/synapse/http/server.py index b6e2ae14a2..f72d986288 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -138,8 +138,8 @@ def wrap_request_handler_with_logging(h): # dispatching to the handler, so that the handler # can update the servlet name in the request # metrics - requests_counter.inc(request.method, - request.request_metrics.name) + requests_counter.labels(request.method, + request.request_metrics.name).inc() yield d return wrapped_request_handler diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 7c680659b6..6fcca5e260 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -22,35 +22,29 @@ from .push_rule_evaluator import PushRuleEvaluatorForEvent from synapse.event_auth import get_user_power_level from synapse.api.constants import EventTypes, Membership -from synapse.metrics import get_metrics_for -from synapse.util.caches import metrics as cache_metrics +from synapse.util.caches import register_cache from synapse.util.caches.descriptors import cached from synapse.util.async import Linearizer from synapse.state import POWER_KEY from collections import namedtuple - +from prometheus_client import Counter logger = logging.getLogger(__name__) rules_by_room = {} -push_metrics = get_metrics_for(__name__) -push_rules_invalidation_counter = push_metrics.register_counter( - "push_rules_invalidation_counter" -) -push_rules_state_size_counter = push_metrics.register_counter( - "push_rules_state_size_counter" -) +push_rules_invalidation_counter = Counter("synapse_push_bulk_push_role_evaluator_push_rules_invalidation_counter", "") +push_rules_state_size_counter = Counter("synapse_push_bulk_push_role_evaluator_push_rules_state_size_counter", "") # Measures whether we use the fast path of using state deltas, or if we have to # recalculate from scratch -push_rules_delta_state_cache_metric = cache_metrics.register_cache( +push_rules_delta_state_cache_metric = register_cache( "cache", - size_callback=lambda: 0, # Meaningless size, as this isn't a cache that stores values - cache_name="push_rules_delta_state_cache_metric", + "push_rules_delta_state_cache_metric", + cache=[], # Meaningless size, as this isn't a cache that stores values ) @@ -64,10 +58,10 @@ class BulkPushRuleEvaluator(object): self.store = hs.get_datastore() self.auth = hs.get_auth() - self.room_push_rule_cache_metrics = cache_metrics.register_cache( + self.room_push_rule_cache_metrics = register_cache( "cache", - size_callback=lambda: 0, # There's not good value for this - cache_name="room_push_rule_cache", + "room_push_rule_cache", + cache=[], # Meaningless size, as this isn't a cache that stores values ) @defer.inlineCallbacks @@ -309,7 +303,7 @@ class RulesForRoom(object): current_state_ids = context.current_state_ids push_rules_delta_state_cache_metric.inc_misses() - push_rules_state_size_counter.inc_by(len(current_state_ids)) + push_rules_state_size_counter.inc(len(current_state_ids)) logger.debug( "Looking for member changes in %r %r", state_group, current_state_ids diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index a41af4fd6c..0e6b1957c6 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -22,20 +22,19 @@ from .streams import STREAMS_MAP, FederationStream from .protocol import ServerReplicationStreamProtocol from synapse.util.metrics import Measure, measure_func +from synapse.metrics import LaterGauge import logging -import synapse.metrics +from prometheus_client import Counter -metrics = synapse.metrics.get_metrics_for(__name__) -stream_updates_counter = metrics.register_counter( - "stream_updates", labels=["stream_name"] +stream_updates_counter = Counter("synapse_replication_tcp_resource_stream_updates", "", ["stream_name"] ) -user_sync_counter = metrics.register_counter("user_sync") -federation_ack_counter = metrics.register_counter("federation_ack") -remove_pusher_counter = metrics.register_counter("remove_pusher") -invalidate_cache_counter = metrics.register_counter("invalidate_cache") -user_ip_cache_counter = metrics.register_counter("user_ip_cache") +user_sync_counter = Counter("synapse_replication_tcp_resource_user_sync", "") +federation_ack_counter = Counter("synapse_replication_tcp_resource_federation_ack", "") +remove_pusher_counter = Counter("synapse_replication_tcp_resource_remove_pusher", "") +invalidate_cache_counter = Counter("synapse_replication_tcp_resource_invalidate_cache", "") +user_ip_cache_counter = Counter("synapse_replication_tcp_resource_user_ip_cache", "") logger = logging.getLogger(__name__) @@ -73,7 +72,8 @@ class ReplicationStreamer(object): # Current connections. self.connections = [] - metrics.register_callback("total_connections", lambda: len(self.connections)) + l = LaterGauge("synapse_replication_tcp_resource_total_connections", "", [], lambda: len(self.connections)) + l.register() # List of streams that clients can subscribe to. # We only support federation stream if federation sending hase been @@ -85,17 +85,15 @@ class ReplicationStreamer(object): self.streams_by_name = {stream.NAME: stream for stream in self.streams} - metrics.register_callback( - "connections_per_stream", + LaterGauge( + "synapse_replication_tcp_resource_connections_per_stream", "", ["stream_name"], lambda: { (stream_name,): len([ conn for conn in self.connections if stream_name in conn.replication_streams ]) for stream_name in self.streams_by_name - }, - labels=["stream_name"], - ) + }).register() self.federation_sender = None if not hs.config.send_federation: @@ -175,7 +173,7 @@ class ReplicationStreamer(object): logger.info( "Streaming: %s -> %s", stream.NAME, updates[-1][0] ) - stream_updates_counter.inc_by(len(updates), stream.NAME) + stream_updates_counter.labels(stream.NAME).inc(len(updates)) # Some streams return multiple rows with the same stream IDs, # we need to make sure they get sent out in batches. We do diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 2262776ab2..d1b625dc30 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -18,8 +18,8 @@ from synapse.api.errors import StoreError from synapse.util.logcontext import LoggingContext, PreserveLoggingContext from synapse.util.caches.descriptors import Cache from synapse.storage.engines import PostgresEngine -import synapse.metrics +from prometheus_client import Histogram from twisted.internet import defer @@ -34,13 +34,10 @@ sql_logger = logging.getLogger("synapse.storage.SQL") transaction_logger = logging.getLogger("synapse.storage.txn") perf_logger = logging.getLogger("synapse.storage.TIME") +sql_scheduling_timer = Histogram("synapse_storage_schedule_time", "") -metrics = synapse.metrics.get_metrics_for("synapse.storage") - -sql_scheduling_timer = metrics.register_distribution("schedule_time") - -sql_query_timer = metrics.register_distribution("query_time", labels=["verb"]) -sql_txn_timer = metrics.register_distribution("transaction_time", labels=["desc"]) +sql_query_timer = Histogram("synapse_storage_query_time", "", ["verb"]) +sql_txn_timer = Histogram("synapse_storage_transaction_time", "", ["desc"]) class LoggingTransaction(object): @@ -117,7 +114,7 @@ class LoggingTransaction(object): finally: msecs = (time.time() * 1000) - start sql_logger.debug("[SQL time] {%s} %f", self.name, msecs) - sql_query_timer.inc_by(msecs, sql.split()[0]) + sql_query_timer.labels(sql.split()[0]).observe(msecs) class PerformanceCounters(object): @@ -287,7 +284,7 @@ class SQLBaseStore(object): self._current_txn_total_time += duration self._txn_perf_counters.update(desc, start, end) - sql_txn_timer.inc_by(duration, desc) + sql_txn_timer.labels(desc).observe(duration) @defer.inlineCallbacks def runInteraction(self, desc, func, *args, **kwargs): @@ -349,7 +346,7 @@ class SQLBaseStore(object): def inner_func(conn, *args, **kwargs): with LoggingContext("runWithConnection") as context: sched_duration_ms = time.time() * 1000 - start_time - sql_scheduling_timer.inc_by(sched_duration_ms) + sql_scheduling_timer.observe(sched_duration_ms) current_context.add_database_scheduled(sched_duration_ms) if self.database_engine.is_connection_closed(conn): diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 05cde96afc..96b48cfdbb 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -40,30 +40,24 @@ import synapse.metrics from synapse.events import EventBase # noqa: F401 from synapse.events.snapshot import EventContext # noqa: F401 +from prometheus_client import Counter + logger = logging.getLogger(__name__) - -metrics = synapse.metrics.get_metrics_for(__name__) -persist_event_counter = metrics.register_counter("persisted_events") -event_counter = metrics.register_counter( - "persisted_events_sep", labels=["type", "origin_type", "origin_entity"] -) +persist_event_counter = Counter("synapse_storage_events_persisted_events", "") +event_counter = Counter("synapse_storage_events_persisted_events_sep", "", ["type", "origin_type", "origin_entity"]) # The number of times we are recalculating the current state -state_delta_counter = metrics.register_counter( - "state_delta", -) +state_delta_counter = Counter("synapse_storage_events_state_delta", "") + # The number of times we are recalculating state when there is only a # single forward extremity -state_delta_single_event_counter = metrics.register_counter( - "state_delta_single_event", -) +state_delta_single_event_counter = Counter("synapse_storage_events_state_delta_single_event", "") + # The number of times we are reculating state when we could have resonably # calculated the delta when we calculated the state for an event we were # persisting. -state_delta_reuse_delta_counter = metrics.register_counter( - "state_delta_reuse_delta", -) +state_delta_reuse_delta_counter = Counter("synapse_storage_events_state_delta_reuse_delta", "") def encode_json(json_object): @@ -445,7 +439,7 @@ class EventsStore(EventsWorkerStore): state_delta_for_room=state_delta_for_room, new_forward_extremeties=new_forward_extremeties, ) - persist_event_counter.inc_by(len(chunk)) + persist_event_counter.inc(len(chunk)) synapse.metrics.event_persisted_position.set( chunk[-1][0].internal_metadata.stream_ordering, ) @@ -460,7 +454,7 @@ class EventsStore(EventsWorkerStore): origin_type = "remote" origin_entity = get_domain_from_id(event.sender) - event_counter.inc(event.type, origin_type, origin_entity) + event_counter.labels(event.type, origin_type, origin_entity).inc() for room_id, new_state in current_state_for_room.iteritems(): self.get_current_state_ids.prefill( diff --git a/tests/metrics/__init__.py b/tests/metrics/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/metrics/test_metric.py b/tests/metrics/test_metric.py deleted file mode 100644 index 069c0be762..0000000000 --- a/tests/metrics/test_metric.py +++ /dev/null @@ -1,192 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2015, 2016 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from tests import unittest - -from synapse.metrics.metric import ( - CounterMetric, CallbackMetric, DistributionMetric, CacheMetric, - _escape_label_value, -) - - -class CounterMetricTestCase(unittest.TestCase): - - def test_scalar(self): - counter = CounterMetric("scalar") - - self.assertEquals(counter.render(), [ - 'scalar 0', - ]) - - counter.inc() - - self.assertEquals(counter.render(), [ - 'scalar 1', - ]) - - counter.inc_by(2) - - self.assertEquals(counter.render(), [ - 'scalar 3' - ]) - - def test_vector(self): - counter = CounterMetric("vector", labels=["method"]) - - # Empty counter doesn't yet know what values it has - self.assertEquals(counter.render(), []) - - counter.inc("GET") - - self.assertEquals(counter.render(), [ - 'vector{method="GET"} 1', - ]) - - counter.inc("GET") - counter.inc("PUT") - - self.assertEquals(counter.render(), [ - 'vector{method="GET"} 2', - 'vector{method="PUT"} 1', - ]) - - -class CallbackMetricTestCase(unittest.TestCase): - - def test_scalar(self): - d = dict() - - metric = CallbackMetric("size", lambda: len(d)) - - self.assertEquals(metric.render(), [ - 'size 0', - ]) - - d["key"] = "value" - - self.assertEquals(metric.render(), [ - 'size 1', - ]) - - def test_vector(self): - vals = dict() - - metric = CallbackMetric("values", lambda: vals, labels=["type"]) - - self.assertEquals(metric.render(), []) - - # Keys have to be tuples, even if they're 1-element - vals[("foo",)] = 1 - vals[("bar",)] = 2 - - self.assertEquals(metric.render(), [ - 'values{type="bar"} 2', - 'values{type="foo"} 1', - ]) - - -class DistributionMetricTestCase(unittest.TestCase): - - def test_scalar(self): - metric = DistributionMetric("thing") - - self.assertEquals(metric.render(), [ - 'thing:count 0', - 'thing:total 0', - ]) - - metric.inc_by(500) - - self.assertEquals(metric.render(), [ - 'thing:count 1', - 'thing:total 500', - ]) - - def test_vector(self): - metric = DistributionMetric("queries", labels=["verb"]) - - self.assertEquals(metric.render(), []) - - metric.inc_by(300, "SELECT") - metric.inc_by(200, "SELECT") - metric.inc_by(800, "INSERT") - - self.assertEquals(metric.render(), [ - 'queries:count{verb="INSERT"} 1', - 'queries:count{verb="SELECT"} 2', - 'queries:total{verb="INSERT"} 800', - 'queries:total{verb="SELECT"} 500', - ]) - - -class CacheMetricTestCase(unittest.TestCase): - - def test_cache(self): - d = dict() - - metric = CacheMetric("cache", lambda: len(d), "cache_name") - - self.assertEquals(metric.render(), [ - 'cache:hits{name="cache_name"} 0', - 'cache:total{name="cache_name"} 0', - 'cache:size{name="cache_name"} 0', - 'cache:evicted_size{name="cache_name"} 0', - ]) - - metric.inc_misses() - d["key"] = "value" - - self.assertEquals(metric.render(), [ - 'cache:hits{name="cache_name"} 0', - 'cache:total{name="cache_name"} 1', - 'cache:size{name="cache_name"} 1', - 'cache:evicted_size{name="cache_name"} 0', - ]) - - metric.inc_hits() - - self.assertEquals(metric.render(), [ - 'cache:hits{name="cache_name"} 1', - 'cache:total{name="cache_name"} 2', - 'cache:size{name="cache_name"} 1', - 'cache:evicted_size{name="cache_name"} 0', - ]) - - metric.inc_evictions(2) - - self.assertEquals(metric.render(), [ - 'cache:hits{name="cache_name"} 1', - 'cache:total{name="cache_name"} 2', - 'cache:size{name="cache_name"} 1', - 'cache:evicted_size{name="cache_name"} 2', - ]) - - -class LabelValueEscapeTestCase(unittest.TestCase): - def test_simple(self): - string = "safjhsdlifhyskljfksdfh" - self.assertEqual(string, _escape_label_value(string)) - - def test_escape(self): - self.assertEqual( - "abc\\\"def\\nghi\\\\", - _escape_label_value("abc\"def\nghi\\"), - ) - - def test_sequence_of_escapes(self): - self.assertEqual( - "abc\\\"def\\nghi\\\\\\n", - _escape_label_value("abc\"def\nghi\\\n"), - ) From fb2806b1868baaa24d0d6e21d10f2c1f6177717e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 22 May 2018 09:31:53 +0100 Subject: [PATCH 197/310] Move in_flight_requests_count to be a callback metric --- synapse/http/request_metrics.py | 39 +++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py index f1e9b41fde..8cb23f4d64 100644 --- a/synapse/http/request_metrics.py +++ b/synapse/http/request_metrics.py @@ -124,12 +124,8 @@ in_flight_requests_db_sched_duration = metrics.register_counter( "in_flight_requests_db_sched_duration_seconds", labels=["method", "servlet"] ) -_in_flight_requests_count = metrics.register_gauge( - "in_flight_requests_count", labels=["method", "servlet"] -) - -# The set of all in flight requests. +# The set of all in flight requests, set[RequestMetrics] _in_flight_requests = set() @@ -138,22 +134,37 @@ def _collect_in_flight(): the in flight request metrics """ - # Map from (method, name) -> int, the number of in flight requests of that - # type - counts = {} - for rm in _in_flight_requests: rm.update_metrics() - key = (rm.method, rm.name,) - counts[key] = counts.get(key, 0) + 1 - - for (method, name), count in counts.iteritems(): - _in_flight_requests_count.set(count, method, name) metrics.register_collector(_collect_in_flight) +def _get_in_flight_counts(): + """Returns a count of all in flight requests by (method, server_name) + + Returns: + dict[tuple[str, str], int] + """ + + # Map from (method, name) -> int, the number of in flight requests of that + # type + counts = {} + for rm in _in_flight_requests: + key = (rm.method, rm.name,) + counts[key] = counts.get(key, 0) + 1 + + return counts + + +metrics.register_callback( + "in_flight_requests_count", + _get_in_flight_counts, + labels=["method", "servlet"] +) + + class RequestMetrics(object): def start(self, time_msec, name, method): self.start = time_msec From c435b0b44172efe485ae1401deb8adca2e774d3c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 22 May 2018 09:34:26 +0100 Subject: [PATCH 198/310] Don't store context --- synapse/http/request_metrics.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py index 8cb23f4d64..5917c83958 100644 --- a/synapse/http/request_metrics.py +++ b/synapse/http/request_metrics.py @@ -230,7 +230,7 @@ class RequestMetrics(object): """Updates the in flight metrics with values from this request. """ - diff = self._request_stats.update() + diff = self._request_stats.update(self.start_context) in_flight_requests_ru_utime.inc_by( diff.ru_utime, self.method, self.name, @@ -258,13 +258,12 @@ class _RequestStats(object): """ __slots__ = [ - "context", "ru_utime", "ru_stime", + "ru_utime", "ru_stime", "db_txn_count", "db_txn_duration_ms", "db_sched_duration_ms", ] - def __init__(self, context, ru_utime, ru_stime, db_txn_count, + def __init__(self, ru_utime, ru_stime, db_txn_count, db_txn_duration_ms, db_sched_duration_ms): - self.context = context self.ru_utime = ru_utime self.ru_stime = ru_stime self.db_txn_count = db_txn_count @@ -276,24 +275,22 @@ class _RequestStats(object): ru_utime, ru_stime = context.get_resource_usage() return _RequestStats( - context, ru_utime, ru_stime, context.db_txn_count, context.db_txn_duration_ms, context.db_sched_duration_ms, ) - def update(self): + def update(self, context): """Updates the current values and returns the difference between the old and new values. Returns: _RequestStats: The difference between the old and new values """ - new = _RequestStats.from_context(self.context) + new = _RequestStats.from_context(context) diff = _RequestStats( - self.context, new.ru_utime - self.ru_utime, new.ru_stime - self.ru_stime, new.db_txn_count - self.db_txn_count, From 020377a5504602cb5d2ce555c77bd18eefc87635 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 22 May 2018 11:16:07 +0100 Subject: [PATCH 199/310] Fix logcontext resource usage tracking --- synapse/util/logcontext.py | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index eab9d57650..0bbda1dcaf 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -60,7 +60,7 @@ class LoggingContext(object): __slots__ = [ "previous_context", "name", "ru_stime", "ru_utime", "db_txn_count", "db_txn_duration_ms", "db_sched_duration_ms", - "usage_start", "usage_end", + "usage_start", "main_thread", "alive", "request", "tag", ] @@ -109,8 +109,10 @@ class LoggingContext(object): # ms spent waiting for db txns to be scheduled self.db_sched_duration_ms = 0 + # If alive has the thread resource usage when the logcontext last + # became active. self.usage_start = None - self.usage_end = None + self.main_thread = threading.current_thread() self.request = None self.tag = "" @@ -159,7 +161,7 @@ class LoggingContext(object): """Restore the logging context in thread local storage to the state it was before this context was entered. Returns: - None to avoid suppressing any exeptions that were thrown. + None to avoid suppressing any exceptions that were thrown. """ current = self.set_current_context(self.previous_context) if current is not self: @@ -185,29 +187,43 @@ class LoggingContext(object): def start(self): if threading.current_thread() is not self.main_thread: + logger.warning("Started logcontext %s on different thread", self) return - if self.usage_start and self.usage_end: - self.ru_utime += self.usage_end.ru_utime - self.usage_start.ru_utime - self.ru_stime += self.usage_end.ru_stime - self.usage_start.ru_stime - self.usage_start = None - self.usage_end = None - + # If we haven't already started record the thread resource usage so + # far if not self.usage_start: self.usage_start = get_thread_resource_usage() def stop(self): if threading.current_thread() is not self.main_thread: + logger.warning("Stopped logcontext %s on different thread", self) return + # When we stop lets record the resource used since we started if self.usage_start: - self.usage_end = get_thread_resource_usage() + usage_end = get_thread_resource_usage() + + self.ru_utime += usage_end.ru_utime - self.usage_start.ru_utime + self.ru_stime += usage_end.ru_stime - self.usage_start.ru_stime + + self.usage_start = None + else: + logger.warning("Called stop on logcontext %s without calling start", self) def get_resource_usage(self): + """Get CPU time used by this logcontext so far. + + Returns: + tuple[float, float]: The user and system CPU usage in seconds + """ ru_utime = self.ru_utime ru_stime = self.ru_stime - if self.usage_start and threading.current_thread() is self.main_thread: + # If we are on the correct thread and we're currently running then we + # can include resource usage so far. + is_main_thread = threading.current_thread() is self.main_thread + if self.alive and self.usage_start and is_main_thread: current = get_thread_resource_usage() ru_utime += current.ru_utime - self.usage_start.ru_utime ru_stime += current.ru_stime - self.usage_start.ru_stime From 7948ecf2342d7053a6ff6daa01268b9331e1c569 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 22 May 2018 11:39:43 +0100 Subject: [PATCH 200/310] Comment --- synapse/util/logcontext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index 0bbda1dcaf..914f616312 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -200,7 +200,7 @@ class LoggingContext(object): logger.warning("Stopped logcontext %s on different thread", self) return - # When we stop lets record the resource used since we started + # When we stop, let's record the resource used since we started if self.usage_start: usage_end = get_thread_resource_usage() From d14d7b8fdc67f8f784c69605f95fe136e032b5d4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 17 May 2018 12:09:18 +0100 Subject: [PATCH 201/310] Rename 'version' param on user consent config we're going to use it for the version we require too. --- synapse/config/consent_config.py | 10 ++++++---- synapse/rest/consent/consent_resource.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py index 675fce0911..45856b9e8a 100644 --- a/synapse/config/consent_config.py +++ b/synapse/config/consent_config.py @@ -18,19 +18,21 @@ from ._base import Config DEFAULT_CONFIG = """\ # User Consent configuration # -# uncomment and configure if enabling the 'consent' resource under 'listeners'. +# Parts of this section are required if enabling the 'consent' resource under +# 'listeners', in particular 'template_dir' and 'version'. # # 'template_dir' gives the location of the templates for the HTML forms. # This directory should contain one subdirectory per language (eg, 'en', 'fr'), # and each language directory should contain the policy document (named as # '.html') and a success page (success.html). # -# 'default_version' gives the version of the policy document to serve up if -# there is no 'v' parameter. +# 'version' specifies the 'current' version of the policy document. It defines +# the version to be served by the consent resource if there is no 'v' +# parameter. # # user_consent: # template_dir: res/templates/privacy -# default_version: 1.0 +# version: 1.0 """ diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py index d791302278..b0536ee375 100644 --- a/synapse/rest/consent/consent_resource.py +++ b/synapse/rest/consent/consent_resource.py @@ -116,7 +116,7 @@ class ConsentResource(Resource): loader = jinja2.FileSystemLoader(consent_template_directory) self._jinja_env = jinja2.Environment(loader=loader) - self._default_consent_verison = consent_config["default_version"] + self._default_consent_verison = consent_config["version"] if hs.config.form_secret is None: raise ConfigError( From 9ea219c5143de009c8e411fbb01da1da5ce50829 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 17 May 2018 17:35:31 +0100 Subject: [PATCH 202/310] Send users a server notice about consent When a user first syncs, we will send them a server notice asking them to consent to the privacy policy if they have not already done so. --- synapse/config/consent_config.py | 8 ++ synapse/handlers/presence.py | 10 +- synapse/replication/tcp/resource.py | 2 + synapse/server.py | 5 + synapse/server.pyi | 4 + .../server_notices/consent_server_notices.py | 101 ++++++++++++++++++ .../server_notices/server_notices_sender.py | 58 ++++++++++ synapse/storage/registration.py | 46 ++++++-- .../add_user_consent_server_notice_sent.sql | 20 ++++ tests/storage/test_registration.py | 11 +- tests/utils.py | 1 + 11 files changed, 255 insertions(+), 11 deletions(-) create mode 100644 synapse/server_notices/consent_server_notices.py create mode 100644 synapse/server_notices/server_notices_sender.py create mode 100644 synapse/storage/schema/delta/49/add_user_consent_server_notice_sent.sql diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py index 45856b9e8a..a6fbc5a058 100644 --- a/synapse/config/consent_config.py +++ b/synapse/config/consent_config.py @@ -30,9 +30,17 @@ DEFAULT_CONFIG = """\ # the version to be served by the consent resource if there is no 'v' # parameter. # +# 'server_notice_content', if enabled, will send a user a "Server Notice" +# asking them to consent to the privacy policy. The 'server_notices' section +# must also be configured for this to work. +# # user_consent: # template_dir: res/templates/privacy # version: 1.0 +# server_notice_content: +# msgtype: m.text +# body: | +# Pls do consent kthx """ diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 91218e40e6..adc816f747 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -87,6 +87,11 @@ assert LAST_ACTIVE_GRANULARITY < IDLE_TIMER class PresenceHandler(object): def __init__(self, hs): + """ + + Args: + hs (synapse.server.HomeServer): + """ self.is_mine = hs.is_mine self.is_mine_id = hs.is_mine_id self.clock = hs.get_clock() @@ -94,8 +99,8 @@ class PresenceHandler(object): self.wheel_timer = WheelTimer() self.notifier = hs.get_notifier() self.federation = hs.get_federation_sender() - self.state = hs.get_state_handler() + self._server_notices_sender = hs.get_server_notices_sender() federation_registry = hs.get_federation_registry() @@ -428,6 +433,9 @@ class PresenceHandler(object): last_user_sync_ts=self.clock.time_msec(), )]) + # send any outstanding server notices to the user. + yield self._server_notices_sender.on_user_syncing(user_id) + @defer.inlineCallbacks def _end(): try: diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index a41af4fd6c..a603c520ea 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -69,6 +69,7 @@ class ReplicationStreamer(object): self.presence_handler = hs.get_presence_handler() self.clock = hs.get_clock() self.notifier = hs.get_notifier() + self._server_notices_sender = hs.get_server_notices_sender() # Current connections. self.connections = [] @@ -253,6 +254,7 @@ class ReplicationStreamer(object): yield self.store.insert_client_ip( user_id, access_token, ip, user_agent, device_id, last_seen, ) + yield self._server_notices_sender.on_user_ip(user_id) def send_sync_to_all_connections(self, data): """Sends a SYNC command to all clients. diff --git a/synapse/server.py b/synapse/server.py index 85f54cd047..e7c733f2d4 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -73,6 +73,7 @@ from synapse.rest.media.v1.media_repository import ( MediaRepositoryResource, ) from synapse.server_notices.server_notices_manager import ServerNoticesManager +from synapse.server_notices.server_notices_sender import ServerNoticesSender from synapse.state import StateHandler, StateResolutionHandler from synapse.storage import DataStore from synapse.streams.events import EventSources @@ -158,6 +159,7 @@ class HomeServer(object): 'room_member_handler', 'federation_registry', 'server_notices_manager', + 'server_notices_sender', ] def __init__(self, hostname, **kwargs): @@ -403,6 +405,9 @@ class HomeServer(object): def build_server_notices_manager(self): return ServerNoticesManager(self) + def build_server_notices_sender(self): + return ServerNoticesSender(self) + def remove_pusher(self, app_id, push_key, user_id): return self.get_pusherpool().remove_pusher(app_id, push_key, user_id) diff --git a/synapse/server.pyi b/synapse/server.pyi index 6fbe15168d..ce28486233 100644 --- a/synapse/server.pyi +++ b/synapse/server.pyi @@ -10,6 +10,7 @@ import synapse.handlers.e2e_keys import synapse.handlers.set_password import synapse.rest.media.v1.media_repository import synapse.server_notices.server_notices_manager +import synapse.server_notices.server_notices_sender import synapse.state import synapse.storage @@ -69,3 +70,6 @@ class HomeServer(object): def get_server_notices_manager(self) -> synapse.server_notices.server_notices_manager.ServerNoticesManager: pass + + def get_server_notices_sender(self) -> synapse.server_notices.server_notices_sender.ServerNoticesSender: + pass diff --git a/synapse/server_notices/consent_server_notices.py b/synapse/server_notices/consent_server_notices.py new file mode 100644 index 0000000000..e9098aef27 --- /dev/null +++ b/synapse/server_notices/consent_server_notices.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from twisted.internet import defer + +from synapse.api.errors import SynapseError +from synapse.config import ConfigError + +logger = logging.getLogger(__name__) + + +class ConsentServerNotices(object): + """Keeps track of whether we need to send users server_notices about + privacy policy consent, and sends one if we do. + """ + def __init__(self, hs): + """ + + Args: + hs (synapse.server.HomeServer): + """ + self._server_notices_manager = hs.get_server_notices_manager() + self._store = hs.get_datastore() + + self._current_consent_version = None + self._server_notice_content = None + self._users_in_progress = set() + + consent_config = hs.config.consent_config + if consent_config is not None: + self._current_consent_version = str(consent_config["version"]) + self._server_notice_content = consent_config.get( + "server_notice_content" + ) + + if self._server_notice_content is not None: + if not self._server_notices_manager.is_enabled(): + raise ConfigError( + "user_consent configuration requires server notices, but " + "server notices are not enabled.", + ) + if 'body' not in self._server_notice_content: + raise ConfigError( + "user_consent server_notice_consent must contain a 'body' " + "key.", + ) + + @defer.inlineCallbacks + def maybe_send_server_notice_to_user(self, user_id): + """Check if we need to send a notice to this user, and does so if so + + Args: + user_id (str): user to check + + Returns: + Deferred + """ + if self._server_notice_content is None: + # not enabled + return + + # make sure we don't send two messages to the same user at once + if user_id in self._users_in_progress: + return + self._users_in_progress.add(user_id) + try: + u = yield self._store.get_user_by_id(user_id) + + if u["consent_version"] == self._current_consent_version: + # user has already consented + return + + if u["consent_server_notice_sent"] == self._current_consent_version: + # we've already sent a notice to the user + return + + # need to send a message + try: + yield self._server_notices_manager.send_notice( + user_id, self._server_notice_content, + ) + yield self._store.user_set_consent_server_notice_sent( + user_id, self._current_consent_version, + ) + except SynapseError as e: + logger.error("Error sending server notice about user consent: %s", e) + finally: + self._users_in_progress.remove(user_id) diff --git a/synapse/server_notices/server_notices_sender.py b/synapse/server_notices/server_notices_sender.py new file mode 100644 index 0000000000..9eade85851 --- /dev/null +++ b/synapse/server_notices/server_notices_sender.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from synapse.server_notices.consent_server_notices import ConsentServerNotices + + +class ServerNoticesSender(object): + """A centralised place which sends server notices automatically when + Certain Events take place + """ + def __init__(self, hs): + """ + + Args: + hs (synapse.server.HomeServer): + """ + # todo: it would be nice to make this more dynamic + self._consent_server_notices = ConsentServerNotices(hs) + + def on_user_syncing(self, user_id): + """Called when the user performs a sync operation. + + This is only called when /sync (or /events) is called on the synapse + master. In a deployment with synchrotrons, on_user_ip is called + + Args: + user_id (str): mxid of user who synced + + Returns: + Deferred + """ + return self._consent_server_notices.maybe_send_server_notice_to_user( + user_id, + ) + + def on_user_ip(self, user_id): + """Called when a worker process saw a client request. + + Args: + user_id (str): mxid + + Returns: + Deferred + """ + return self._consent_server_notices.maybe_send_server_notice_to_user( + user_id, + ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 8d1a01f1ee..a530e29f43 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -33,7 +33,10 @@ class RegistrationWorkerStore(SQLBaseStore): keyvalues={ "name": user_id, }, - retcols=["name", "password_hash", "is_guest"], + retcols=[ + "name", "password_hash", "is_guest", + "consent_version", "consent_server_notice_sent", + ], allow_none=True, desc="get_user_by_id", ) @@ -297,12 +300,41 @@ class RegistrationStore(RegistrationWorkerStore, Raises: StoreError(404) if user not found """ - return self._simple_update_one( - table='users', - keyvalues={'name': user_id, }, - updatevalues={'consent_version': consent_version, }, - desc="user_set_consent_version" - ) + def f(txn): + self._simple_update_one_txn( + txn, + table='users', + keyvalues={'name': user_id, }, + updatevalues={'consent_version': consent_version, }, + ) + self._invalidate_cache_and_stream( + txn, self.get_user_by_id, (user_id,) + ) + return self.runInteraction("user_set_consent_version", f) + + def user_set_consent_server_notice_sent(self, user_id, consent_version): + """Updates the user table to record that we have sent the user a server + notice about privacy policy consent + + Args: + user_id (str): full mxid of the user to update + consent_version (str): version of the policy we have notified the + user about + + Raises: + StoreError(404) if user not found + """ + def f(txn): + self._simple_update_one_txn( + txn, + table='users', + keyvalues={'name': user_id, }, + updatevalues={'consent_server_notice_sent': consent_version, }, + ) + self._invalidate_cache_and_stream( + txn, self.get_user_by_id, (user_id,) + ) + return self.runInteraction("user_set_consent_server_notice_sent", f) def user_delete_access_tokens(self, user_id, except_token_id=None, device_id=None): diff --git a/synapse/storage/schema/delta/49/add_user_consent_server_notice_sent.sql b/synapse/storage/schema/delta/49/add_user_consent_server_notice_sent.sql new file mode 100644 index 0000000000..14dcf18d73 --- /dev/null +++ b/synapse/storage/schema/delta/49/add_user_consent_server_notice_sent.sql @@ -0,0 +1,20 @@ +/* Copyright 2018 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* record whether we have sent a server notice about consenting to the + * privacy policy. Specifically records the version of the policy we sent + * a message about. + */ +ALTER TABLE users ADD COLUMN consent_server_notice_sent TEXT; diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index 7c7b164ee6..cc637dda1c 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -42,9 +42,14 @@ class RegistrationStoreTestCase(unittest.TestCase): yield self.store.register(self.user_id, self.tokens[0], self.pwhash) self.assertEquals( - # TODO(paul): Surely this field should be 'user_id', not 'name' - # Additionally surely it shouldn't come in a 1-element list - {"name": self.user_id, "password_hash": self.pwhash, "is_guest": 0}, + { + # TODO(paul): Surely this field should be 'user_id', not 'name' + "name": self.user_id, + "password_hash": self.pwhash, + "is_guest": 0, + "consent_version": None, + "consent_server_notice_sent": None, + }, (yield self.store.get_user_by_id(self.user_id)) ) diff --git a/tests/utils.py b/tests/utils.py index c2beb5d9f7..63d8e9c640 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -63,6 +63,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): config.federation_rc_concurrent = 10 config.filter_timeline_limit = 5000 config.user_directory_search_all_users = False + config.consent_config = None # disable user directory updates, because they get done in the # background, which upsets the test runner. From d5dca9a04fbac7fea0822eaf2fd513b13449184c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 18 May 2018 15:41:40 +0100 Subject: [PATCH 203/310] Move consent config parsing into ConsentConfig turns out we need to reuse this, so it's better in the config class. --- synapse/config/consent_config.py | 16 +++++++++++++++- synapse/rest/consent/consent_resource.py | 8 +++----- synapse/server_notices/consent_server_notices.py | 10 ++-------- tests/utils.py | 2 +- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py index a6fbc5a058..8698b2993f 100644 --- a/synapse/config/consent_config.py +++ b/synapse/config/consent_config.py @@ -45,8 +45,22 @@ DEFAULT_CONFIG = """\ class ConsentConfig(Config): + def __init__(self): + super(ConsentConfig, self).__init__() + + self.user_consent_version = None + self.user_consent_template_dir = None + self.user_consent_server_notice_content = None + def read_config(self, config): - self.consent_config = config.get("user_consent") + consent_config = config.get("user_consent") + if consent_config is None: + return + self.user_consent_version = str(consent_config["version"]) + self.user_consent_template_dir = consent_config["template_dir"] + self.user_consent_server_notice_content = consent_config.get( + "server_notice_content", + ) def default_config(self, **kwargs): return DEFAULT_CONFIG diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py index b0536ee375..306385601c 100644 --- a/synapse/rest/consent/consent_resource.py +++ b/synapse/rest/consent/consent_resource.py @@ -95,8 +95,8 @@ class ConsentResource(Resource): # this is required by the request_handler wrapper self.clock = hs.get_clock() - consent_config = hs.config.consent_config - if consent_config is None: + self._default_consent_verison = hs.config.user_consent_version + if self._default_consent_verison is None: raise ConfigError( "Consent resource is enabled but user_consent section is " "missing in config file.", @@ -104,7 +104,7 @@ class ConsentResource(Resource): # daemonize changes the cwd to /, so make the path absolute now. consent_template_directory = path.abspath( - consent_config["template_dir"], + hs.config.user_consent_template_dir, ) if not path.isdir(consent_template_directory): raise ConfigError( @@ -116,8 +116,6 @@ class ConsentResource(Resource): loader = jinja2.FileSystemLoader(consent_template_directory) self._jinja_env = jinja2.Environment(loader=loader) - self._default_consent_verison = consent_config["version"] - if hs.config.form_secret is None: raise ConfigError( "Consent resource is enabled but form_secret is not set in " diff --git a/synapse/server_notices/consent_server_notices.py b/synapse/server_notices/consent_server_notices.py index e9098aef27..440f6b1cd4 100644 --- a/synapse/server_notices/consent_server_notices.py +++ b/synapse/server_notices/consent_server_notices.py @@ -35,16 +35,10 @@ class ConsentServerNotices(object): self._server_notices_manager = hs.get_server_notices_manager() self._store = hs.get_datastore() - self._current_consent_version = None - self._server_notice_content = None self._users_in_progress = set() - consent_config = hs.config.consent_config - if consent_config is not None: - self._current_consent_version = str(consent_config["version"]) - self._server_notice_content = consent_config.get( - "server_notice_content" - ) + self._current_consent_version = hs.config.user_consent_version + self._server_notice_content = hs.config.user_consent_server_notice_content if self._server_notice_content is not None: if not self._server_notices_manager.is_enabled(): diff --git a/tests/utils.py b/tests/utils.py index 63d8e9c640..9626e6fd78 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -63,7 +63,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): config.federation_rc_concurrent = 10 config.filter_timeline_limit = 5000 config.user_directory_search_all_users = False - config.consent_config = None + config.user_consent_server_notice_content = None # disable user directory updates, because they get done in the # background, which upsets the test runner. From 8810685df936ba60e0e9d36c9371c20f305c9126 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 22 May 2018 10:57:56 +0100 Subject: [PATCH 204/310] Stub out ServerNoticesSender on the workers ... and have the sync endpoints call it directly rather than obsure indirection via PresenceHandler --- synapse/handlers/events.py | 5 ++ synapse/handlers/presence.py | 4 -- synapse/rest/client/v2_alpha/sync.py | 4 ++ synapse/server.py | 7 +++ .../server_notices/server_notices_sender.py | 8 ++-- .../worker_server_notices_sender.py | 46 +++++++++++++++++++ 6 files changed, 66 insertions(+), 8 deletions(-) create mode 100644 synapse/server_notices/worker_server_notices_sender.py diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index d3685fb12a..8bc642675f 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -48,6 +48,7 @@ class EventStreamHandler(BaseHandler): self.notifier = hs.get_notifier() self.state = hs.get_state_handler() + self._server_notices_sender = hs.get_server_notices_sender() @defer.inlineCallbacks @log_function @@ -58,6 +59,10 @@ class EventStreamHandler(BaseHandler): If `only_keys` is not None, events from keys will be sent down. """ + + # send any outstanding server notices to the user. + yield self._server_notices_sender.on_user_syncing(auth_user_id) + auth_user = UserID.from_string(auth_user_id) presence_handler = self.hs.get_presence_handler() diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index adc816f747..500a131874 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -100,7 +100,6 @@ class PresenceHandler(object): self.notifier = hs.get_notifier() self.federation = hs.get_federation_sender() self.state = hs.get_state_handler() - self._server_notices_sender = hs.get_server_notices_sender() federation_registry = hs.get_federation_registry() @@ -433,9 +432,6 @@ class PresenceHandler(object): last_user_sync_ts=self.clock.time_msec(), )]) - # send any outstanding server notices to the user. - yield self._server_notices_sender.on_user_syncing(user_id) - @defer.inlineCallbacks def _end(): try: diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index eb91c0b293..a291cffbf1 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -85,6 +85,7 @@ class SyncRestServlet(RestServlet): self.clock = hs.get_clock() self.filtering = hs.get_filtering() self.presence_handler = hs.get_presence_handler() + self._server_notices_sender = hs.get_server_notices_sender() @defer.inlineCallbacks def on_GET(self, request): @@ -149,6 +150,9 @@ class SyncRestServlet(RestServlet): else: since_token = None + # send any outstanding server notices to the user. + yield self._server_notices_sender.on_user_syncing(user.to_string()) + affect_presence = set_presence != PresenceState.OFFLINE if affect_presence: diff --git a/synapse/server.py b/synapse/server.py index e7c733f2d4..58dbf78437 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -74,6 +74,9 @@ from synapse.rest.media.v1.media_repository import ( ) from synapse.server_notices.server_notices_manager import ServerNoticesManager from synapse.server_notices.server_notices_sender import ServerNoticesSender +from synapse.server_notices.worker_server_notices_sender import ( + WorkerServerNoticesSender, +) from synapse.state import StateHandler, StateResolutionHandler from synapse.storage import DataStore from synapse.streams.events import EventSources @@ -403,9 +406,13 @@ class HomeServer(object): return FederationHandlerRegistry() def build_server_notices_manager(self): + if self.config.worker_app: + raise Exception("Workers cannot send server notices") return ServerNoticesManager(self) def build_server_notices_sender(self): + if self.config.worker_app: + return WorkerServerNoticesSender(self) return ServerNoticesSender(self) def remove_pusher(self, app_id, push_key, user_id): diff --git a/synapse/server_notices/server_notices_sender.py b/synapse/server_notices/server_notices_sender.py index 9eade85851..5d23965f34 100644 --- a/synapse/server_notices/server_notices_sender.py +++ b/synapse/server_notices/server_notices_sender.py @@ -31,9 +31,6 @@ class ServerNoticesSender(object): def on_user_syncing(self, user_id): """Called when the user performs a sync operation. - This is only called when /sync (or /events) is called on the synapse - master. In a deployment with synchrotrons, on_user_ip is called - Args: user_id (str): mxid of user who synced @@ -45,7 +42,7 @@ class ServerNoticesSender(object): ) def on_user_ip(self, user_id): - """Called when a worker process saw a client request. + """Called on the master when a worker process saw a client request. Args: user_id (str): mxid @@ -53,6 +50,9 @@ class ServerNoticesSender(object): Returns: Deferred """ + # The synchrotrons use a stubbed version of ServerNoticesSender, so + # we check for notices to send to the user in on_user_ip as well as + # in on_user_syncing return self._consent_server_notices.maybe_send_server_notice_to_user( user_id, ) diff --git a/synapse/server_notices/worker_server_notices_sender.py b/synapse/server_notices/worker_server_notices_sender.py new file mode 100644 index 0000000000..25abb4ccf5 --- /dev/null +++ b/synapse/server_notices/worker_server_notices_sender.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from twisted.internet import defer + + +class WorkerServerNoticesSender(object): + """Stub impl of ServerNoticesSender which does nothing""" + def __init__(self, hs): + """ + Args: + hs (synapse.server.HomeServer): + """ + + def on_user_syncing(self, user_id): + """Called when the user performs a sync operation. + + Args: + user_id (str): mxid of user who synced + + Returns: + Deferred + """ + return defer.succeed() + + def on_user_ip(self, user_id): + """Called on the master when a worker process saw a client request. + + Args: + user_id (str): mxid + + Returns: + Deferred + """ + raise AssertionError("on_user_ip unexpectedly called on worker") From a5e2941aad9acdd5033709807bb6ddd40e4435eb Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 22 May 2018 08:56:52 +0100 Subject: [PATCH 205/310] Reject attempts to send event before privacy consent is given Returns an M_CONSENT_NOT_GIVEN error (cf https://github.com/matrix-org/matrix-doc/issues/1252) if consent is not yet given. --- synapse/api/errors.py | 30 ++++++++++- synapse/api/urls.py | 50 +++++++++++++++++++ synapse/config/consent_config.py | 10 ++++ synapse/handlers/message.py | 86 +++++++++++++++++++++++++++++++- synapse/handlers/room.py | 4 ++ tests/utils.py | 1 + 6 files changed, 179 insertions(+), 2 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index a9ff5576f3..aca804280c 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -19,6 +19,7 @@ import logging import simplejson as json from six import iteritems +from six.moves import http_client logger = logging.getLogger(__name__) @@ -51,6 +52,7 @@ class Codes(object): THREEPID_DENIED = "M_THREEPID_DENIED" INVALID_USERNAME = "M_INVALID_USERNAME" SERVER_NOT_TRUSTED = "M_SERVER_NOT_TRUSTED" + CONSENT_NOT_GIVEN = "M_CONSENT_NOT_GIVEN" class CodeMessageException(RuntimeError): @@ -138,6 +140,32 @@ class SynapseError(CodeMessageException): return res +class ConsentNotGivenError(SynapseError): + """The error returned to the client when the user has not consented to the + privacy policy. + """ + def __init__(self, msg, consent_uri): + """Constructs a ConsentNotGivenError + + Args: + msg (str): The human-readable error message + consent_url (str): The URL where the user can give their consent + """ + super(ConsentNotGivenError, self).__init__( + code=http_client.FORBIDDEN, + msg=msg, + errcode=Codes.CONSENT_NOT_GIVEN + ) + self._consent_uri = consent_uri + + def error_dict(self): + return cs_error( + self.msg, + self.errcode, + consent_uri=self._consent_uri + ) + + class RegistrationError(SynapseError): """An error raised when a registration event fails.""" pass @@ -292,7 +320,7 @@ def cs_error(msg, code=Codes.UNKNOWN, **kwargs): Args: msg (str): The error message. - code (int): The error code. + code (str): The error code. kwargs : Additional keys to add to the response. Returns: A dict representing the error response JSON. diff --git a/synapse/api/urls.py b/synapse/api/urls.py index 91a33a3402..bb46b5da8a 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,6 +15,12 @@ # limitations under the License. """Contains the URL paths to prefix various aspects of the server with. """ +from hashlib import sha256 +import hmac + +from six.moves.urllib.parse import urlencode + +from synapse.config import ConfigError CLIENT_PREFIX = "/_matrix/client/api/v1" CLIENT_V2_ALPHA_PREFIX = "/_matrix/client/v2_alpha" @@ -25,3 +32,46 @@ SERVER_KEY_PREFIX = "/_matrix/key/v1" SERVER_KEY_V2_PREFIX = "/_matrix/key/v2" MEDIA_PREFIX = "/_matrix/media/r0" LEGACY_MEDIA_PREFIX = "/_matrix/media/v1" + + +class ConsentURIBuilder(object): + def __init__(self, hs_config): + """ + Args: + hs_config (synapse.config.homeserver.HomeServerConfig): + """ + if hs_config.form_secret is None: + raise ConfigError( + "form_secret not set in config", + ) + if hs_config.public_baseurl is None: + raise ConfigError( + "public_baseurl not set in config", + ) + + self._hmac_secret = hs_config.form_secret.encode("utf-8") + self._public_baseurl = hs_config.public_baseurl + + def build_user_consent_uri(self, user_id): + """Build a URI which we can give to the user to do their privacy + policy consent + + Args: + user_id (str): mxid or username of user + + Returns + (str) the URI where the user can do consent + """ + mac = hmac.new( + key=self._hmac_secret, + msg=user_id, + digestmod=sha256, + ).hexdigest() + consent_uri = "%s_matrix/consent?%s" % ( + self._public_baseurl, + urlencode({ + "u": user_id, + "h": mac + }), + ) + return consent_uri diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py index 8698b2993f..44c4711e6c 100644 --- a/synapse/config/consent_config.py +++ b/synapse/config/consent_config.py @@ -34,6 +34,10 @@ DEFAULT_CONFIG = """\ # asking them to consent to the privacy policy. The 'server_notices' section # must also be configured for this to work. # +# 'block_events_error', if set, will block any attempts to send events +# until the user consents to the privacy policy. The value of the setting is +# used as the text of the error. +# # user_consent: # template_dir: res/templates/privacy # version: 1.0 @@ -41,6 +45,8 @@ DEFAULT_CONFIG = """\ # msgtype: m.text # body: | # Pls do consent kthx +# block_events_error: | +# You can't send any messages until you consent to the privacy policy. """ @@ -51,6 +57,7 @@ class ConsentConfig(Config): self.user_consent_version = None self.user_consent_template_dir = None self.user_consent_server_notice_content = None + self.block_events_without_consent_error = None def read_config(self, config): consent_config = config.get("user_consent") @@ -61,6 +68,9 @@ class ConsentConfig(Config): self.user_consent_server_notice_content = consent_config.get( "server_notice_content", ) + self.block_events_without_consent_error = consent_config.get( + "block_events_error", + ) def default_config(self, **kwargs): return DEFAULT_CONFIG diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 8343b5839d..c3adbc6c95 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -20,10 +20,15 @@ import sys from canonicaljson import encode_canonical_json import six from twisted.internet import defer, reactor +from twisted.internet.defer import succeed from twisted.python.failure import Failure from synapse.api.constants import EventTypes, Membership, MAX_DEPTH -from synapse.api.errors import AuthError, Codes, SynapseError +from synapse.api.errors import ( + AuthError, Codes, SynapseError, + ConsentNotGivenError, +) +from synapse.api.urls import ConsentURIBuilder from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.events.utils import serialize_event from synapse.events.validator import EventValidator @@ -431,6 +436,9 @@ class EventCreationHandler(object): self.spam_checker = hs.get_spam_checker() + if self.config.block_events_without_consent_error is not None: + self._consent_uri_builder = ConsentURIBuilder(self.config) + @defer.inlineCallbacks def create_event(self, requester, event_dict, token_id=None, txn_id=None, prev_events_and_hashes=None): @@ -482,6 +490,10 @@ class EventCreationHandler(object): target, e ) + is_exempt = yield self._is_exempt_from_privacy_policy(builder) + if not is_exempt: + yield self.assert_accepted_privacy_policy(requester) + if token_id is not None: builder.internal_metadata.token_id = token_id @@ -496,6 +508,78 @@ class EventCreationHandler(object): defer.returnValue((event, context)) + def _is_exempt_from_privacy_policy(self, builder): + """"Determine if an event to be sent is exempt from having to consent + to the privacy policy + + Args: + builder (synapse.events.builder.EventBuilder): event being created + + Returns: + Deferred[bool]: true if the event can be sent without the user + consenting + """ + # the only thing the user can do is join the server notices room. + if builder.type == EventTypes.Member: + membership = builder.content.get("membership", None) + if membership == Membership.JOIN: + return self._is_server_notices_room(builder.room_id) + return succeed(False) + + @defer.inlineCallbacks + def _is_server_notices_room(self, room_id): + if self.config.server_notices_mxid is None: + defer.returnValue(False) + user_ids = yield self.store.get_users_in_room(room_id) + defer.returnValue(self.config.server_notices_mxid in user_ids) + + @defer.inlineCallbacks + def assert_accepted_privacy_policy(self, requester): + """Check if a user has accepted the privacy policy + + Called when the given user is about to do something that requires + privacy consent. We see if the user is exempt and otherwise check that + they have given consent. If they have not, a ConsentNotGiven error is + raised. + + Args: + requester (synapse.types.Requester): + The user making the request + + Returns: + Deferred[None]: returns normally if the user has consented or is + exempt + + Raises: + ConsentNotGivenError: if the user has not given consent yet + """ + if self.config.block_events_without_consent_error is None: + return + + # exempt AS users from needing consent + if requester.app_service is not None: + return + + user_id = requester.user.to_string() + + # exempt the system notices user + if ( + self.config.server_notices_mxid is not None and + user_id == self.config.server_notices_mxid + ): + return + + u = yield self.store.get_user_by_id(user_id) + assert u is not None + if u["consent_version"] == self.config.user_consent_version: + return + + consent_uri = self._consent_uri_builder.build_user_consent_uri(user_id) + raise ConsentNotGivenError( + msg=self.config.block_events_without_consent_error, + consent_uri=consent_uri, + ) + @defer.inlineCallbacks def send_nonmember_event(self, requester, event, context, ratelimit=True): """ diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 5e9fa95a2d..b5850db42f 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -126,6 +126,10 @@ class RoomCreationHandler(BaseHandler): except Exception: raise SynapseError(400, "Invalid user_id: %s" % (i,)) + yield self.event_creation_handler.assert_accepted_privacy_policy( + requester, + ) + invite_3pid_list = config.get("invite_3pid", []) visibility = config.get("visibility", None) diff --git a/tests/utils.py b/tests/utils.py index 9626e6fd78..262c4a5714 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -64,6 +64,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): config.filter_timeline_limit = 5000 config.user_directory_search_all_users = False config.user_consent_server_notice_content = None + config.block_events_without_consent_error = None # disable user directory updates, because they get done in the # background, which upsets the test runner. From b5b2d5d64b2690e250fb56e851d5556c09d13f86 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 22 May 2018 14:00:23 +0100 Subject: [PATCH 206/310] Fix dependency on jinja2 Delay the import of ConsentResource, so that we can get away without jinja2 if people don't have the consent resource enabled. Fixes #3259 --- synapse/app/homeserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index bceb21a8d5..caccbaa814 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -41,7 +41,6 @@ from synapse.python_dependencies import CONDITIONAL_REQUIREMENTS, \ from synapse.replication.http import ReplicationRestResource, REPLICATION_PREFIX from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory from synapse.rest import ClientRestResource -from synapse.rest.consent.consent_resource import ConsentResource from synapse.rest.key.v1.server_key_resource import LocalKey from synapse.rest.key.v2 import KeyApiV2Resource from synapse.rest.media.v0.content_repository import ContentRepoResource @@ -186,6 +185,7 @@ class SynapseHomeServer(HomeServer): }) if name == "consent": + from synapse.rest.consent.consent_resource import ConsentResource consent_resource = ConsentResource(self) if compress: consent_resource = gz_wrap(consent_resource) From 669400e22f514efbd846cad1f2943a48d79276b7 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 22 May 2018 14:18:53 +0100 Subject: [PATCH 207/310] Enable auto-escaping for the consent templates ... to reduce the risk of somebody introducing an html injection attack... --- synapse/rest/consent/consent_resource.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py index 306385601c..e6a6dcbefa 100644 --- a/synapse/rest/consent/consent_resource.py +++ b/synapse/rest/consent/consent_resource.py @@ -114,7 +114,10 @@ class ConsentResource(Resource): ) loader = jinja2.FileSystemLoader(consent_template_directory) - self._jinja_env = jinja2.Environment(loader=loader) + self._jinja_env = jinja2.Environment( + loader=loader, + autoescape=jinja2.select_autoescape(['html', 'htm', 'xml']), + ) if hs.config.form_secret is None: raise ConfigError( From 7b36d06a69942653f6c6fbb6dbf341a452002237 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 22 May 2018 14:50:22 +0100 Subject: [PATCH 208/310] Add a 'has_consented' template var to consent forms fixes #3260 --- docs/privacy_policy_templates/README.md | 2 +- docs/privacy_policy_templates/en/1.0.html | 6 ++++++ synapse/rest/consent/consent_resource.py | 17 ++++++++++++++--- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/docs/privacy_policy_templates/README.md b/docs/privacy_policy_templates/README.md index 8e91c516b3..a3e6fc0986 100644 --- a/docs/privacy_policy_templates/README.md +++ b/docs/privacy_policy_templates/README.md @@ -9,7 +9,7 @@ form_secret: user_consent: template_dir: docs/privacy_policy_templates - default_version: 1.0 + version: 1.0 ``` You should then be able to enable the `consent` resource under a `listener` diff --git a/docs/privacy_policy_templates/en/1.0.html b/docs/privacy_policy_templates/en/1.0.html index ab8666f0c3..55c5e4b612 100644 --- a/docs/privacy_policy_templates/en/1.0.html +++ b/docs/privacy_policy_templates/en/1.0.html @@ -4,6 +4,11 @@ Matrix.org Privacy policy + {% if has_consented %} +

+ Your base already belong to us. +

+ {% else %}

All your base are belong to us.

@@ -13,5 +18,6 @@ + {% endif %} diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py index e6a6dcbefa..724911d1e6 100644 --- a/synapse/rest/consent/consent_resource.py +++ b/synapse/rest/consent/consent_resource.py @@ -95,8 +95,8 @@ class ConsentResource(Resource): # this is required by the request_handler wrapper self.clock = hs.get_clock() - self._default_consent_verison = hs.config.user_consent_version - if self._default_consent_verison is None: + self._default_consent_version = hs.config.user_consent_version + if self._default_consent_version is None: raise ConfigError( "Consent resource is enabled but user_consent section is " "missing in config file.", @@ -132,6 +132,7 @@ class ConsentResource(Resource): return NOT_DONE_YET @wrap_html_request_handler + @defer.inlineCallbacks def _async_render_GET(self, request): """ Args: @@ -139,16 +140,26 @@ class ConsentResource(Resource): """ version = parse_string(request, "v", - default=self._default_consent_verison) + default=self._default_consent_version) username = parse_string(request, "u", required=True) userhmac = parse_string(request, "h", required=True) self._check_hash(username, userhmac) + if username.startswith('@'): + qualified_user_id = username + else: + qualified_user_id = UserID(username, self.hs.hostname).to_string() + + u = yield self.store.get_user_by_id(qualified_user_id) + if u is None: + raise NotFoundError("Unknown user") + try: self._render_template( request, "%s.html" % (version,), user=username, userhmac=userhmac, version=version, + has_consented=(u["consent_version"] == version), ) except TemplateNotFound: raise NotFoundError("Unknown policy version") From cb2a2ad791155bd1e1ef58caa2ef2894a74fab08 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 22 May 2018 16:23:39 +0100 Subject: [PATCH 209/310] get_domains_from_state returns list of tuples --- synapse/handlers/federation.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index ba3ede8024..c85d194511 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -751,6 +751,16 @@ class FederationHandler(BaseHandler): curr_state = yield self.state_handler.get_current_state(room_id) def get_domains_from_state(state): + """Get joined domains from state + + Args: + state (dict[tuple, FrozenEvent]): State map from type/state + key to event. + + Returns: + list[tuple[str, int]]: Returns a list of servers with the + lowest depth of their joins. Sorted by lowest depth first. + """ joined_users = [ (state_key, int(event.depth)) for (e_type, state_key), event in state.items() @@ -861,13 +871,13 @@ class FederationHandler(BaseHandler): likely_domains = get_domains_from_state(states[e_id]) success = yield try_backfill([ - dom for dom in likely_domains + dom for dom, _ in likely_domains if dom not in tried_domains ]) if success: defer.returnValue(True) - tried_domains.update(likely_domains) + tried_domains.update(dom for dom, _ in likely_domains) defer.returnValue(False) From 8f5a688d420c8f6b51826c561da9094b76fbea1e Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 22 May 2018 10:56:03 -0500 Subject: [PATCH 210/310] cleanups, self-registration --- synapse/handlers/presence.py | 7 ++++--- synapse/http/request_metrics.py | 15 +++------------ synapse/replication/tcp/resource.py | 9 +++++---- 3 files changed, 12 insertions(+), 19 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index aca822c46a..4ee87d5714 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -142,8 +142,9 @@ class PresenceHandler(object): } LaterGauge( - "user_to_current_state_size", "", [], lambda: len(self.user_to_current_state) - ).register() + "synapse_handlers_presence_user_to_current_state_size", "", [], + lambda: len(self.user_to_current_state) + ) now = self.clock.time_msec() for state in active_presence: @@ -212,7 +213,7 @@ class PresenceHandler(object): 60 * 1000, ) - LaterGauge("wheel_timer_size", "", [], lambda: len(self.wheel_timer)).register() + LaterGauge("synapse_handlers_presence_wheel_timer_size", "", [], lambda: len(self.wheel_timer)) @defer.inlineCallbacks def _on_shutdown(self): diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py index 0984870e7e..e7f1bfc4ae 100644 --- a/synapse/http/request_metrics.py +++ b/synapse/http/request_metrics.py @@ -71,24 +71,14 @@ in_flight_requests_db_sched_duration = Counter("synapse_http_request_in_flight_r _in_flight_requests = set() -def _collect_in_flight(): - """Called just before metrics are collected, so we use it to update all - the in flight request metrics - """ - - for rm in _in_flight_requests: - rm.update_metrics() - - -metrics.register_collector(_collect_in_flight) - - def _get_in_flight_counts(): """Returns a count of all in flight requests by (method, server_name) Returns: dict[tuple[str, str], int] """ + for rm in _in_flight_requests: + rm.update_metrics() # Map from (method, name) -> int, the number of in flight requests of that # type @@ -99,6 +89,7 @@ def _get_in_flight_counts(): return counts + LaterGauge( "synapse_http_request_metrics_in_flight_requests_count", "", ["method", "servlet"], diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index ed251c79ac..1ca30bc31a 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -73,8 +73,8 @@ class ReplicationStreamer(object): # Current connections. self.connections = [] - l = LaterGauge("synapse_replication_tcp_resource_total_connections", "", [], lambda: len(self.connections)) - l.register() + LaterGauge("synapse_replication_tcp_resource_total_connections", "", [], + lambda: len(self.connections)) # List of streams that clients can subscribe to. # We only support federation stream if federation sending hase been @@ -87,14 +87,15 @@ class ReplicationStreamer(object): self.streams_by_name = {stream.NAME: stream for stream in self.streams} LaterGauge( - "synapse_replication_tcp_resource_connections_per_stream", "", ["stream_name"], + "synapse_replication_tcp_resource_connections_per_stream", "", + ["stream_name"], lambda: { (stream_name,): len([ conn for conn in self.connections if stream_name in conn.replication_streams ]) for stream_name in self.streams_by_name - }).register() + }) self.federation_sender = None if not hs.config.send_federation: From 07bb9bdae82b8f1345be7c60475be3edac64b336 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 22 May 2018 10:57:28 -0500 Subject: [PATCH 211/310] update gitignore to remove some files that got put on my machine --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 7acfe56d26..7940158c5b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.pyc .*.swp +*~ .DS_Store _trial_temp/ @@ -13,6 +14,7 @@ docs/build/ cmdclient_config.json homeserver*.db homeserver*.log +homeserver*.log.* homeserver*.pid homeserver*.yaml @@ -40,6 +42,7 @@ media_store/ *.tac build/ +venv/ localhost-800*/ static/client/register/register_config.js From 0a078026eae694337cfdf8c9460d1b231b1bd04a Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 22 May 2018 17:14:06 +0100 Subject: [PATCH 212/310] comment typo --- synapse/handlers/deactivate_account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 4eb18775e8..d58ea6c650 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -61,7 +61,7 @@ class DeactivateAccountHandler(BaseHandler): yield self.store.user_delete_threepids(user_id) yield self.store.user_set_password_hash(user_id, None) - # Add the user to a table of users penpding deactivation (ie. + # Add the user to a table of users pending deactivation (ie. # removal from all the rooms they're a member of) yield self.store.add_user_pending_deactivation(user_id) From 08bfc48abf8c138b540ede6c816338c044130dec Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 22 May 2018 17:27:03 +0100 Subject: [PATCH 213/310] custom error code for not leaving server notices room --- synapse/api/errors.py | 1 + synapse/handlers/room_member.py | 1 + 2 files changed, 2 insertions(+) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index aca804280c..e6ad3768f0 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -53,6 +53,7 @@ class Codes(object): INVALID_USERNAME = "M_INVALID_USERNAME" SERVER_NOT_TRUSTED = "M_SERVER_NOT_TRUSTED" CONSENT_NOT_GIVEN = "M_CONSENT_NOT_GIVEN" + CANNOT_LEAVE_SERVER_NOTICE_ROOM = "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM" class CodeMessageException(RuntimeError): diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 5263f09aaa..82adfc8fdf 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -306,6 +306,7 @@ class RoomMemberHandler(object): raise SynapseError( http_client.FORBIDDEN, "You cannot leave this room", + errcode=Codes.CANNOT_LEAVE_SERVER_NOTICE_ROOM, ) if effective_membership_state == Membership.INVITE: From ad2823ee27d7bfe766c8b84efbd0bcde02962048 Mon Sep 17 00:00:00 2001 From: hera Date: Tue, 22 May 2018 16:38:46 +0000 Subject: [PATCH 214/310] fix synchrotron --- synapse/server_notices/worker_server_notices_sender.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/server_notices/worker_server_notices_sender.py b/synapse/server_notices/worker_server_notices_sender.py index 25abb4ccf5..4a133026c3 100644 --- a/synapse/server_notices/worker_server_notices_sender.py +++ b/synapse/server_notices/worker_server_notices_sender.py @@ -32,7 +32,7 @@ class WorkerServerNoticesSender(object): Returns: Deferred """ - return defer.succeed() + return defer.succeed(None) def on_user_ip(self, user_id): """Called on the master when a worker process saw a client request. From d8cb7225d2902ae3dd7fbcd1b3b2ebd084d81ca4 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Tue, 22 May 2018 18:09:09 +0100 Subject: [PATCH 215/310] daily user type phone home stats --- synapse/app/homeserver.py | 4 ++++ synapse/storage/registration.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index caccbaa814..026422a023 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -434,6 +434,10 @@ def run(hs): total_nonbridged_users = yield hs.get_datastore().count_nonbridged_users() stats["total_nonbridged_users"] = total_nonbridged_users + daily_user_type_results = yield hs.get_datastore().count_daily_user_type() + for name, count in daily_user_type_results.iteritems(): + stats["daily_user_type_" + name] = count + room_count = yield hs.get_datastore().get_room_count() stats["total_room_count"] = room_count diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index a530e29f43..d8e60d2e87 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -485,6 +485,35 @@ class RegistrationStore(RegistrationWorkerStore, ret = yield self.runInteraction("count_users", _count_users) defer.returnValue(ret) + def count_daily_user_type(self): + """ + Counts 1) native non guest users + 2) native guests users + 3) bridged users + who registered on the homeserver in the past 24 hours + """ + def _count_daily_user_type(txn): + yesterday = int(self._clock.time()) - (60 * 60 * 24) + + sql = """ + SELECT user_type, COALESCE(count(*), 0) AS count FROM ( + SELECT + CASE + WHEN is_guest=0 AND appservice_id IS NULL THEN 'native' + WHEN is_guest=1 AND appservice_id IS NULL THEN 'guest' + WHEN is_guest=0 AND appservice_id IS NOT NULL THEN 'bridged' + END AS user_type + FROM users + WHERE creation_ts > ? + ) AS t GROUP BY user_type + """ + results = {'native': 0, 'guest': 0, 'bridged': 0} + txn.execute(sql, (yesterday,)) + for row in txn: + results[row[0]] = row[1] + return results + return self.runInteraction("count_daily_user_type", _count_daily_user_type) + @defer.inlineCallbacks def count_nonbridged_users(self): def _count_users(txn): From 5494c1d71e47245b9750fc260462fe4c2ca3e273 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 22 May 2018 18:15:21 +0100 Subject: [PATCH 216/310] Don't support limitless pagination The pagination storage function supported not specifiying a limit on the number of events returned. This was triggered when using the search or context API with a limit of zero, which the storage function took to mean not being limited. --- synapse/storage/stream.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index ea24710ad8..1701d03866 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -684,8 +684,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): results to only those before direction(char): Either 'b' or 'f' to indicate whether we are paginating forwards or backwards from `from_key`. - limit (int): The maximum number of events to return. Zero or less - means no limit. + limit (int): The maximum number of events to return. event_filter (Filter|None): If provided filters the events to those that match the filter. @@ -694,6 +693,9 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): as a list of _EventDictReturn and a token that points to the end of the result set. """ + + assert int(limit) >= 0 + # Tokens really represent positions between elements, but we use # the convention of pointing to the event before the gap. Hence # we have a bit of asymmetry when it comes to equalities. @@ -723,18 +725,14 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): bounds += " AND " + filter_clause args.extend(filter_args) - if int(limit) > 0: - args.append(int(limit)) - limit_str = " LIMIT ?" - else: - limit_str = "" + args.append(int(limit)) sql = ( "SELECT event_id, topological_ordering, stream_ordering" " FROM events" " WHERE outlier = ? AND room_id = ? AND %(bounds)s" " ORDER BY topological_ordering %(order)s," - " stream_ordering %(order)s %(limit)s" + " stream_ordering %(order)s LIMIT ?" ) % { "bounds": bounds, "order": order, From a17e901f4dd1825d4bf21e55488939d95e1d7d80 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 22 May 2018 18:24:32 +0100 Subject: [PATCH 217/310] Remove unused string formatting param --- synapse/storage/stream.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 1701d03866..fb463c525a 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -736,7 +736,6 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): ) % { "bounds": bounds, "order": order, - "limit": limit_str } txn.execute(sql, args) From 586b66b1977e2c678e23a3b2d25a5b699b637a8e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 22 May 2018 19:00:48 +0100 Subject: [PATCH 218/310] Fix that states is a dict of dicts --- synapse/handlers/federation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index c85d194511..6a381f0023 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -853,10 +853,13 @@ class FederationHandler(BaseHandler): [resolve(room_id, [e]) for e in event_ids], consumeErrors=True, )) + + # dict[str, dict[tuple, str]], a map from event_id to state map of + # event_ids. states = dict(zip(event_ids, [s.state for s in states])) state_map = yield self.store.get_events( - [e_id for ids in states.values() for e_id in ids], + [e_id for ids in states.values() for e_id in ids.itervalues()], get_prev_content=False ) states = { From e85b5a0ff717e0388179ad9664a8fdfca4b61d9a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 22 May 2018 19:02:48 +0100 Subject: [PATCH 219/310] Use iter* methods --- synapse/handlers/federation.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 6a381f0023..786c200378 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -763,7 +763,7 @@ class FederationHandler(BaseHandler): """ joined_users = [ (state_key, int(event.depth)) - for (e_type, state_key), event in state.items() + for (e_type, state_key), event in state.iteritems() if e_type == EventTypes.Member and event.membership == Membership.JOIN ] @@ -780,7 +780,7 @@ class FederationHandler(BaseHandler): except Exception: pass - return sorted(joined_domains.items(), key=lambda d: d[1]) + return sorted(joined_domains.iteritems(), key=lambda d: d[1]) curr_domains = get_domains_from_state(curr_state) @@ -797,7 +797,7 @@ class FederationHandler(BaseHandler): yield self.backfill( dom, room_id, limit=100, - extremities=[e for e in extremities.keys()] + extremities=extremities, ) # If this succeeded then we probably already have the # appropriate stuff. @@ -843,7 +843,7 @@ class FederationHandler(BaseHandler): tried_domains = set(likely_domains) tried_domains.add(self.server_name) - event_ids = list(extremities.keys()) + event_ids = list(extremities.iterkeys()) logger.debug("calling resolve_state_groups in _maybe_backfill") resolve = logcontext.preserve_fn( @@ -859,15 +859,15 @@ class FederationHandler(BaseHandler): states = dict(zip(event_ids, [s.state for s in states])) state_map = yield self.store.get_events( - [e_id for ids in states.values() for e_id in ids.itervalues()], + [e_id for ids in states.itervalues() for e_id in ids.itervalues()], get_prev_content=False ) states = { key: { k: state_map[e_id] - for k, e_id in state_dict.items() + for k, e_id in state_dict.iteritems() if e_id in state_map - } for key, state_dict in states.items() + } for key, state_dict in states.iteritems() } for e_id, _ in sorted_extremeties_tuple: From 228f1f584ef100f18440362303775f7c860c01ca Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 22 May 2018 15:02:38 -0500 Subject: [PATCH 220/310] fix the test failures --- synapse/replication/tcp/protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py index 5848f57c5e..bdb110663c 100644 --- a/synapse/replication/tcp/protocol.py +++ b/synapse/replication/tcp/protocol.py @@ -513,7 +513,7 @@ class ClientReplicationStreamProtocol(BaseReplicationStreamProtocol): def on_RDATA(self, cmd): stream_name = cmd.stream_name - inbound_rdata_count.inc(stream_name) + inbound_rdata_count.labels(stream_name).inc() try: row = STREAMS_MAP[stream_name].ROW_TYPE(*cmd.row) From 85ba83eb5100abf02cf373d9a8d5010526facd45 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 22 May 2018 16:28:23 -0500 Subject: [PATCH 221/310] fixes --- synapse/app/homeserver.py | 6 ++-- synapse/federation/transaction_queue.py | 6 ++-- synapse/metrics/__init__.py | 12 ++++++-- synapse/notifier.py | 6 ++-- synapse/push/httppusher.py | 4 +-- synapse/util/caches/__init__.py | 40 ++++++++++++++++++------- synapse/util/caches/descriptors.py | 2 +- 7 files changed, 52 insertions(+), 24 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index a5b135193f..449bfacdb9 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -34,6 +34,7 @@ from synapse.module_api import ModuleApi from synapse.http.additional_resource import AdditionalResource from synapse.http.server import RootRedirect from synapse.http.site import SynapseSite +from synapse.metrics import RegistryProxy from synapse.metrics.resource import METRICS_PREFIX from synapse.python_dependencies import CONDITIONAL_REQUIREMENTS, \ check_requirements @@ -60,6 +61,8 @@ from twisted.web.resource import EncodingResourceWrapper, NoResource from twisted.web.server import GzipEncoderFactory from twisted.web.static import File +from prometheus_client.twisted import MetricsResource + logger = logging.getLogger("synapse.app.homeserver") @@ -229,8 +232,7 @@ class SynapseHomeServer(HomeServer): resources[WEB_CLIENT_PREFIX] = build_resource_for_web_client(self) if name == "metrics" and self.get_config().enable_metrics: - from prometheus_client.twisted import MetricsResource - resources[METRICS_PREFIX] = MetricsResource() + resources[METRICS_PREFIX] = MetricsResource(RegistryProxy()) if name == "replication": resources[REPLICATION_PREFIX] = ReplicationRestResource(self) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 778924a13c..2049351fdd 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -64,7 +64,7 @@ class TransactionQueue(object): # done self.pending_transactions = {} - LaterGauge("pending_destinations", "", [], + LaterGauge("synapse_federation_client_pending_destinations", "", [], lambda: len(self.pending_transactions), ) @@ -89,11 +89,11 @@ class TransactionQueue(object): self.pending_edus_keyed_by_dest = edus_keyed = {} LaterGauge( - "pending_pdus", "", [], + "synapse_federation_client_pending_pdus", "", [], lambda: sum(map(len, pdus.values())), ) LaterGauge( - "pending_edus", "", [], + "synapse_federation_client_pending_edus", "", [], lambda: ( sum(map(len, edus.values())) + sum(map(len, presence.values())) diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index ab0b921497..38408efb54 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -29,12 +29,20 @@ from twisted.internet import reactor logger = logging.getLogger(__name__) - running_on_pypy = platform.python_implementation() == 'PyPy' all_metrics = [] all_collectors = [] all_gauges = {} + +class RegistryProxy(object): + + def collect(self): + for metric in REGISTRY.collect(): + if not metric.name.startswith("__"): + yield metric + + @attr.s(hash=True) class LaterGauge(object): @@ -45,7 +53,7 @@ class LaterGauge(object): def collect(self): - g = GaugeMetricFamily(self.name, self.desc, self.labels) + g = GaugeMetricFamily(self.name, self.desc, labels=self.labels) try: calls = self.caller() diff --git a/synapse/notifier.py b/synapse/notifier.py index 123e6f1840..40cc553918 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -194,14 +194,14 @@ class Notifier(object): all_user_streams.add(x) return sum(stream.count_listeners() for stream in all_user_streams) - LaterGauge("listeners", "", [], count_listeners) + LaterGauge("synapse_notifier_listeners", "", [], count_listeners) LaterGauge( - "rooms", "", [], + "synapse_notifier_rooms", "", [], lambda: count(bool, self.room_to_user_streams.values()), ) LaterGauge( - "users", "", [], + "synapse_notifier_users", "", [], lambda: len(self.user_to_user_stream), ) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index e22088ad6f..bf7ff74a1a 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -28,9 +28,9 @@ from prometheus_client import Counter logger = logging.getLogger(__name__) -http_push_processed_counter = Counter("http_pushes_processed", "") +http_push_processed_counter = Counter("synapse_http_httppusher_http_pushes_processed", "") -http_push_failed_counter = Counter("http_pushes_failed", "") +http_push_failed_counter = Counter("synapse_http_httppusher_http_pushes_failed", "") class HttpPusher(object): diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index 438dcddf55..1c511a7072 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from prometheus_client.core import GaugeMetricFamily, REGISTRY +from prometheus_client.core import Gauge, REGISTRY, GaugeMetricFamily import os @@ -22,10 +22,20 @@ CACHE_SIZE_FACTOR = float(os.environ.get("SYNAPSE_CACHE_FACTOR", 0.5)) caches_by_name = {} collectors_by_name = {} -def register_cache(name, cache_name, cache): +cache_size = Gauge("synapse_util_caches_cache:size", "", ["name"]) +cache_hits = Gauge("synapse_util_caches_cache:hits", "", ["name"]) +cache_evicted = Gauge("synapse_util_caches_cache:evicted_size", "", ["name"]) +cache_total = Gauge("synapse_util_caches_cache:total", "", ["name"]) + +response_cache_size = Gauge("synapse_util_caches_response_cache:size", "", ["name"]) +response_cache_hits = Gauge("synapse_util_caches_response_cache:hits", "", ["name"]) +response_cache_evicted = Gauge("synapse_util_caches_response_cache:evicted_size", "", ["name"]) +response_cache_total = Gauge("synapse_util_caches_response_cache:total", "", ["name"]) + +def register_cache(cache_type, cache_name, cache): # Check if the metric is already registered. Unregister it, if so. - metric_name = "synapse_util_caches_%s:%s" % (name, cache_name,) + metric_name = "cache_%s_%s" % (cache_type, cache_name,) if metric_name in collectors_by_name.keys(): REGISTRY.unregister(collectors_by_name[metric_name]) @@ -44,15 +54,22 @@ def register_cache(name, cache_name, cache): def inc_evictions(self, size=1): self.evicted_size += size - def collect(self): - cache_size = len(cache) + def describe(self): + return [] - gm = GaugeMetricFamily(metric_name, "", labels=["size", "hits", "misses", "total"]) - gm.add_metric(["size"], cache_size) - gm.add_metric(["hits"], self.hits) - gm.add_metric(["misses"], self.misses) - gm.add_metric(["total"], self.hits + self.misses) - yield gm + def collect(self): + if cache_type == "response_cache": + response_cache_size.labels(cache_name).set(len(cache)) + response_cache_hits.labels(cache_name).set(self.hits) + response_cache_evicted.labels(cache_name).set(self.evicted_size) + response_cache_total.labels(cache_name).set(self.hits + self.misses) + else: + cache_size.labels(cache_name).set(len(cache)) + cache_hits.labels(cache_name).set(self.hits) + cache_evicted.labels(cache_name).set(self.evicted_size) + cache_total.labels(cache_name).set(self.hits + self.misses) + + yield GaugeMetricFamily("__unused", "") metric = CacheMetric() REGISTRY.register(metric) @@ -60,6 +77,7 @@ def register_cache(name, cache_name, cache): collectors_by_name[metric_name] = metric return metric + KNOWN_KEYS = { key: key for key in ( diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py index a4188eb099..8a9dcb2fc2 100644 --- a/synapse/util/caches/descriptors.py +++ b/synapse/util/caches/descriptors.py @@ -80,7 +80,7 @@ class Cache(object): self.name = name self.keylen = keylen self.thread = None - self.metrics = register_cache("descriptor", name, self.cache) + self.metrics = register_cache("cache", name, self.cache) def _on_evicted(self, evicted_count): self.metrics.inc_evictions(evicted_count) From 4abeaedcf333ee688f8a39efe9520bc3e2e54770 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 22 May 2018 16:45:11 -0500 Subject: [PATCH 222/310] tests/metrics is gone now --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 89ca6d7a19..3e114e382c 100644 --- a/tox.ini +++ b/tox.ini @@ -51,7 +51,7 @@ usedevelop=true commands = /usr/bin/find "{toxinidir}" -name '*.pyc' -delete coverage run {env:COVERAGE_OPTS:} --source="{toxinidir}/synapse" \ - "{envbindir}/trial" {env:TRIAL_FLAGS:} {posargs:tests/metrics tests/config} \ + "{envbindir}/trial" {env:TRIAL_FLAGS:} {posargs:tests/config} \ {env:TOXSUFFIX:} {env:DUMP_COVERAGE_COMMAND:coverage report -m} From 071206304d088aac8bd0e2fff600141dae1d09b6 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 22 May 2018 16:54:22 -0500 Subject: [PATCH 223/310] cleanup pep8 errors --- synapse/federation/federation_server.py | 5 +- synapse/federation/transaction_queue.py | 22 +++-- synapse/http/request_metrics.py | 114 ++++++++++++++++++------ synapse/notifier.py | 3 +- synapse/util/caches/__init__.py | 7 +- 5 files changed, 114 insertions(+), 37 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 8211273006..2d420a58a2 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -46,10 +46,13 @@ received_pdus_counter = Counter("synapse_federation_server_received_pdus", "") received_edus_counter = Counter("synapse_federation_server_received_edus", "") -received_queries_counter = Counter("synapse_federation_server_received_queries", "", ["type"]) +received_queries_counter = Counter( + "synapse_federation_server_received_queries", "", ["type"] +) class FederationServer(FederationBase): + def __init__(self, hs): super(FederationServer, self).__init__(hs) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 2049351fdd..53442688c8 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -28,7 +28,10 @@ from synapse.handlers.presence import format_user_presence_state, get_interested import synapse.metrics from synapse.metrics import LaterGauge from synapse.metrics import ( - sent_edus_counter, sent_transactions_counter, events_processed_counter) + sent_edus_counter, + sent_transactions_counter, + events_processed_counter, +) from prometheus_client import Counter @@ -37,7 +40,9 @@ import logging logger = logging.getLogger(__name__) -sent_pdus_destination_dist = Counter("synapse_federation_client_sent_pdu_destinations", "") +sent_pdus_destination_dist = Counter( + "synapse_federation_client_sent_pdu_destinations", "" +) class TransactionQueue(object): @@ -64,7 +69,10 @@ class TransactionQueue(object): # done self.pending_transactions = {} - LaterGauge("synapse_federation_client_pending_destinations", "", [], + LaterGauge( + "synapse_federation_client_pending_destinations", + "", + [], lambda: len(self.pending_transactions), ) @@ -89,11 +97,15 @@ class TransactionQueue(object): self.pending_edus_keyed_by_dest = edus_keyed = {} LaterGauge( - "synapse_federation_client_pending_pdus", "", [], + "synapse_federation_client_pending_pdus", + "", + [], lambda: sum(map(len, pdus.values())), ) LaterGauge( - "synapse_federation_client_pending_edus", "", [], + "synapse_federation_client_pending_edus", + "", + [], lambda: ( sum(map(len, edus.values())) + sum(map(len, presence.values())) diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py index e7f1bfc4ae..7f11b5c5a4 100644 --- a/synapse/http/request_metrics.py +++ b/synapse/http/request_metrics.py @@ -25,47 +25,87 @@ logger = logging.getLogger(__name__) # total number of responses served, split by method/servlet/tag -response_count = Counter("synapse_http_server_response_count", "", ["method", "servlet", "tag"]) +response_count = Counter( + "synapse_http_server_response_count", "", ["method", "servlet", "tag"] +) -requests_counter = Counter("synapse_http_server_requests_received", "", ["method", "servlet"]) +requests_counter = Counter( + "synapse_http_server_requests_received", "", ["method", "servlet"] +) -outgoing_responses_counter = Counter("synapse_http_server_responses", "", ["method", "code"]) +outgoing_responses_counter = Counter( + "synapse_http_server_responses", "", ["method", "code"] +) -response_timer = Histogram("synapse_http_server_response_time_seconds", "", ["method", "servlet", "tag"]) +response_timer = Histogram( + "synapse_http_server_response_time_seconds", "", ["method", "servlet", "tag"] +) -response_ru_utime = Counter("synapse_http_server_response_ru_utime_seconds", "", ["method", "servlet", "tag"]) +response_ru_utime = Counter( + "synapse_http_server_response_ru_utime_seconds", "", ["method", "servlet", "tag"] +) -response_ru_stime = Counter("synapse_http_server_response_ru_stime_seconds", "", ["method", "servlet", "tag"]) +response_ru_stime = Counter( + "synapse_http_server_response_ru_stime_seconds", "", ["method", "servlet", "tag"] +) -response_db_txn_count = Counter("synapse_http_server_response_db_txn_count", "", ["method", "servlet", "tag"]) +response_db_txn_count = Counter( + "synapse_http_server_response_db_txn_count", "", ["method", "servlet", "tag"] +) # seconds spent waiting for db txns, excluding scheduling time, when processing # this request -response_db_txn_duration = Counter("synapse_http_server_response_db_txn_duration_seconds", "", ["method", "servlet", "tag"]) +response_db_txn_duration = Counter( + "synapse_http_server_response_db_txn_duration_seconds", + "", + ["method", "servlet", "tag"], +) # seconds spent waiting for a db connection, when processing this request -response_db_sched_duration = Counter("synapse_http_request_response_db_sched_duration_seconds", "", ["method", "servlet", "tag"] +response_db_sched_duration = Counter( + "synapse_http_request_response_db_sched_duration_seconds", + "", + ["method", "servlet", "tag"], ) # size in bytes of the response written -response_size = Counter("synapse_http_request_response_size", "", ["method", "servlet", "tag"] +response_size = Counter( + "synapse_http_request_response_size", "", ["method", "servlet", "tag"] ) # In flight metrics are incremented while the requests are in flight, rather # than when the response was written. -in_flight_requests_ru_utime = Counter("synapse_http_request_in_flight_requests_ru_utime_seconds", "", ["method", "servlet"]) +in_flight_requests_ru_utime = Counter( + "synapse_http_request_in_flight_requests_ru_utime_seconds", + "", + ["method", "servlet"], +) -in_flight_requests_ru_stime = Counter("synapse_http_request_in_flight_requests_ru_stime_seconds", "", ["method", "servlet"]) +in_flight_requests_ru_stime = Counter( + "synapse_http_request_in_flight_requests_ru_stime_seconds", + "", + ["method", "servlet"], +) -in_flight_requests_db_txn_count = Counter("synapse_http_request_in_flight_requests_db_txn_count", "", ["method", "servlet"]) +in_flight_requests_db_txn_count = Counter( + "synapse_http_request_in_flight_requests_db_txn_count", "", ["method", "servlet"] +) # seconds spent waiting for db txns, excluding scheduling time, when processing # this request -in_flight_requests_db_txn_duration = Counter("synapse_http_request_in_flight_requests_db_txn_duration_seconds", "", ["method", "servlet"]) +in_flight_requests_db_txn_duration = Counter( + "synapse_http_request_in_flight_requests_db_txn_duration_seconds", + "", + ["method", "servlet"], +) # seconds spent waiting for a db connection, when processing this request -in_flight_requests_db_sched_duration = Counter("synapse_http_request_in_flight_requests_db_sched_duration_seconds", "", ["method", "servlet"]) +in_flight_requests_db_sched_duration = Counter( + "synapse_http_request_in_flight_requests_db_sched_duration_seconds", + "", + ["method", "servlet"], +) # The set of all in flight requests, set[RequestMetrics] _in_flight_requests = set() @@ -91,9 +131,10 @@ def _get_in_flight_counts(): LaterGauge( - "synapse_http_request_metrics_in_flight_requests_count", "", + "synapse_http_request_metrics_in_flight_requests_count", + "", ["method", "servlet"], - _get_in_flight_counts + _get_in_flight_counts, ) @@ -128,16 +169,23 @@ class RequestMetrics(object): response_count.labels(request.method, self.name, tag).inc() - response_timer.labels(request.method, self.name, tag).observe(time_msec - self.start) + response_timer.labels(request.method, self.name, tag).observe( + time_msec - self.start + ) ru_utime, ru_stime = context.get_resource_usage() response_ru_utime.labels(request.method, self.name, tag).inc(ru_utime) response_ru_stime.labels(request.method, self.name, tag).inc(ru_stime) - response_db_txn_count.labels(request.method, self.name, tag).inc(context.db_txn_count) - response_db_txn_duration.labels(request.method, self.name, tag).inc(context.db_txn_duration_ms / 1000.) + response_db_txn_count.labels(request.method, self.name, tag).inc( + context.db_txn_count + ) + response_db_txn_duration.labels(request.method, self.name, tag).inc( + context.db_txn_duration_ms / 1000. + ) response_db_sched_duration.labels(request.method, self.name, tag).inc( - context.db_sched_duration_ms / 1000.) + context.db_sched_duration_ms / 1000. + ) response_size.labels(request.method, self.name, tag).inc(request.sentLength) @@ -154,11 +202,17 @@ class RequestMetrics(object): in_flight_requests_ru_utime.labels(self.method, self.name).inc(diff.ru_utime) in_flight_requests_ru_stime.labels(self.method, self.name).inc(diff.ru_stime) - in_flight_requests_db_txn_count.labels(self.method, self.name).inc(diff.db_txn_count) + in_flight_requests_db_txn_count.labels(self.method, self.name).inc( + diff.db_txn_count + ) - in_flight_requests_db_txn_duration.labels(self.method, self.name).inc(diff.db_txn_duration_ms / 1000.) + in_flight_requests_db_txn_duration.labels(self.method, self.name).inc( + diff.db_txn_duration_ms / 1000. + ) - in_flight_requests_db_sched_duration.labels(self.method, self.name).inc(diff.db_sched_duration_ms / 1000.) + in_flight_requests_db_sched_duration.labels(self.method, self.name).inc( + diff.db_sched_duration_ms / 1000. + ) class _RequestStats(object): @@ -166,12 +220,16 @@ class _RequestStats(object): """ __slots__ = [ - "ru_utime", "ru_stime", - "db_txn_count", "db_txn_duration_ms", "db_sched_duration_ms", + "ru_utime", + "ru_stime", + "db_txn_count", + "db_txn_duration_ms", + "db_sched_duration_ms", ] - def __init__(self, ru_utime, ru_stime, db_txn_count, - db_txn_duration_ms, db_sched_duration_ms): + def __init__( + self, ru_utime, ru_stime, db_txn_count, db_txn_duration_ms, db_sched_duration_ms + ): self.ru_utime = ru_utime self.ru_stime = ru_stime self.db_txn_count = db_txn_count diff --git a/synapse/notifier.py b/synapse/notifier.py index 40cc553918..6dce20a284 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -40,7 +40,8 @@ logger = logging.getLogger(__name__) notified_events_counter = Counter("synapse_notifier_notified_events", "") -users_woken_by_stream_counter = Counter("synapse_notifier_users_woken_by_stream", "", ["stream"]) +users_woken_by_stream_counter = Counter( + "synapse_notifier_users_woken_by_stream", "", ["stream"]) # TODO(paul): Should be shared somewhere diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index 1c511a7072..e0c22df249 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -29,13 +29,16 @@ cache_total = Gauge("synapse_util_caches_cache:total", "", ["name"]) response_cache_size = Gauge("synapse_util_caches_response_cache:size", "", ["name"]) response_cache_hits = Gauge("synapse_util_caches_response_cache:hits", "", ["name"]) -response_cache_evicted = Gauge("synapse_util_caches_response_cache:evicted_size", "", ["name"]) +response_cache_evicted = Gauge( + "synapse_util_caches_response_cache:evicted_size", "", ["name"] +) response_cache_total = Gauge("synapse_util_caches_response_cache:total", "", ["name"]) + def register_cache(cache_type, cache_name, cache): # Check if the metric is already registered. Unregister it, if so. - metric_name = "cache_%s_%s" % (cache_type, cache_name,) + metric_name = "cache_%s_%s" % (cache_type, cache_name) if metric_name in collectors_by_name.keys(): REGISTRY.unregister(collectors_by_name[metric_name]) From 53cc2cde1f609ec34a4ce6a7c678302c65ddfe53 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 22 May 2018 17:32:57 -0500 Subject: [PATCH 224/310] cleanup --- synapse/federation/send_queue.py | 5 ++--- synapse/handlers/appservice.py | 9 +++++--- synapse/handlers/presence.py | 12 ++++++---- synapse/http/client.py | 3 ++- synapse/http/matrixfederationclient.py | 6 +++-- synapse/metrics/__init__.py | 28 +++++++++++++++++++----- synapse/push/bulk_push_rule_evaluator.py | 6 +++-- synapse/storage/events.py | 9 +++++--- synapse/util/metrics.py | 15 ++++++++----- 9 files changed, 64 insertions(+), 29 deletions(-) diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py index e6e1888f3a..c7ed465617 100644 --- a/synapse/federation/send_queue.py +++ b/synapse/federation/send_queue.py @@ -74,9 +74,8 @@ class FederationRemoteSendQueue(object): # lambda binds to the queue rather than to the name of the queue which # changes. ARGH. def register(name, queue): - LaterGauge("synapse_federation_send_queue_%s_size" % (queue_name,), "", - lambda: len(queue), - ) + LaterGauge("synapse_federation_send_queue_%s_size" % (queue_name,), + "", lambda: len(queue)) for queue_name in [ "presence_map", "presence_changed", "keyed_edu", "keyed_edu_changed", diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index a7345331af..d9f35a5dba 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -127,12 +127,15 @@ class ApplicationServicesHandler(object): now = self.clock.time_msec() ts = yield self.store.get_received_ts(events[-1].event_id) - synapse.metrics.event_processing_positions.labels("appservice_sender").set(upper_bound) + synapse.metrics.event_processing_positions.labels( + "appservice_sender").set(upper_bound) events_processed_counter.inc(len(events)) - synapse.metrics.event_processing_lag.labels("appservice_sender").set(now - ts) - synapse.metrics.event_processing_last_ts.labels("appservice_sender").set(ts) + synapse.metrics.event_processing_lag.labels( + "appservice_sender").set(now - ts) + synapse.metrics.event_processing_last_ts.labels( + "appservice_sender").set(ts) finally: self.is_processing = False diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 4ee87d5714..12939aa507 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -47,7 +47,8 @@ logger = logging.getLogger(__name__) notified_presence_counter = Counter("synapse_handler_presence_notified_presence", "") -federation_presence_out_counter = Counter("synapse_handler_presence_federation_presence_out", "") +federation_presence_out_counter = Counter( + "synapse_handler_presence_federation_presence_out", "") presence_updates_counter = Counter("synapse_handler_presence_presence_updates", "") timers_fired_counter = Counter("synapse_handler_presence_timers_fired", "") federation_presence_counter = Counter("synapse_handler_presence_federation_presence", "") @@ -55,8 +56,10 @@ bump_active_time_counter = Counter("synapse_handler_presence_bump_active_time", get_updates_counter = Counter("synapse_handler_presence_get_updates", "", ["type"]) -notify_reason_counter = Counter("synapse_handler_presence_notify_reason", "", ["reason"]) -state_transition_counter = Counter("synapse_handler_presence_state_transition", "", ["from", "to"] +notify_reason_counter = Counter( + "synapse_handler_presence_notify_reason", "", ["reason"]) +state_transition_counter = Counter( + "synapse_handler_presence_state_transition", "", ["from", "to"] ) @@ -213,7 +216,8 @@ class PresenceHandler(object): 60 * 1000, ) - LaterGauge("synapse_handlers_presence_wheel_timer_size", "", [], lambda: len(self.wheel_timer)) + LaterGauge("synapse_handlers_presence_wheel_timer_size", "", [], + lambda: len(self.wheel_timer)) @defer.inlineCallbacks def _on_shutdown(self): diff --git a/synapse/http/client.py b/synapse/http/client.py index 61a1d2e2b3..4d4eee3d64 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -50,7 +50,8 @@ import urllib logger = logging.getLogger(__name__) outgoing_requests_counter = Counter("synapse_http_client_requests", "", ["method"]) -incoming_responses_counter = Counter("synapse_http_client_responses", "", ["method", "code"]) +incoming_responses_counter = Counter("synapse_http_client_responses", "", + ["method", "code"]) class SimpleHttpClient(object): diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 259d3884e2..77eaa06a1a 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -48,8 +48,10 @@ from prometheus_client import Counter logger = logging.getLogger(__name__) outbound_logger = logging.getLogger("synapse.http.outbound") -outgoing_requests_counter = Counter("synapse_http_matrixfederationclient_requests", "", ["method"]) -incoming_responses_counter = Counter("synapse_http_matrixfederationclient_responses", "", ["method", "code"]) +outgoing_requests_counter = Counter("synapse_http_matrixfederationclient_requests", + "", ["method"]) +incoming_responses_counter = Counter("synapse_http_matrixfederationclient_responses", + "", ["method", "code"]) MAX_LONG_RETRIES = 10 diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index 38408efb54..bed37b5f56 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -21,15 +21,14 @@ import platform import attr from prometheus_client import Gauge, Histogram, Counter -from prometheus_client.core import ( - GaugeMetricFamily, CounterMetricFamily, REGISTRY) +from prometheus_client.core import GaugeMetricFamily, CounterMetricFamily, REGISTRY from twisted.internet import reactor logger = logging.getLogger(__name__) -running_on_pypy = platform.python_implementation() == 'PyPy' +running_on_pypy = platform.python_implementation() == "PyPy" all_metrics = [] all_collectors = [] all_gauges = {} @@ -87,9 +86,16 @@ class LaterGauge(object): # gc_unreachable = Gauge("python_gc_unreachable_total", "Unreachable GC objects", ["gen"]) -gc_time = Histogram("python_gc_time", "Time taken to GC (ms)", ["gen"], buckets=[1, 2, 5, 10, 25, 50, 100, 250, 500, 1000]) +gc_time = Histogram( + "python_gc_time", + "Time taken to GC (ms)", + ["gen"], + buckets=[1, 2, 5, 10, 25, 50, 100, 250, 500, 1000], +) + class GCCounts(object): + def collect(self): gc_counts = gc.get_count() @@ -99,14 +105,23 @@ class GCCounts(object): yield cm + REGISTRY.register(GCCounts()) # # Twisted reactor metrics # -tick_time = Histogram("python_twisted_reactor_tick_time", "Tick time of the Twisted reactor (ms)", buckets=[1, 2, 5, 10, 50, 100, 250, 500, 1000, 2000]) -pending_calls_metric = Histogram("python_twisted_reactor_pending_calls", "Pending calls", buckets=[1, 2, 5, 10, 25, 50, 100, 250, 500, 1000]) +tick_time = Histogram( + "python_twisted_reactor_tick_time", + "Tick time of the Twisted reactor (ms)", + buckets=[1, 2, 5, 10, 50, 100, 250, 500, 1000, 2000], +) +pending_calls_metric = Histogram( + "python_twisted_reactor_pending_calls", + "Pending calls", + buckets=[1, 2, 5, 10, 25, 50, 100, 250, 500, 1000], +) # # Federation Metrics @@ -134,6 +149,7 @@ event_processing_last_ts = Gauge("synapse_event_processing_last_ts", "", ["name" # finished being processed. event_processing_lag = Gauge("synapse_event_processing_lag", "", ["name"]) + def runUntilCurrentTimer(func): @functools.wraps(func) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 6fcca5e260..b0053e7f3f 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -36,8 +36,10 @@ logger = logging.getLogger(__name__) rules_by_room = {} -push_rules_invalidation_counter = Counter("synapse_push_bulk_push_role_evaluator_push_rules_invalidation_counter", "") -push_rules_state_size_counter = Counter("synapse_push_bulk_push_role_evaluator_push_rules_state_size_counter", "") +push_rules_invalidation_counter = Counter( + "synapse_push_bulk_push_role_evaluator_push_rules_invalidation_counter", "") +push_rules_state_size_counter = Counter( + "synapse_push_bulk_push_role_evaluator_push_rules_state_size_counter", "") # Measures whether we use the fast path of using state deltas, or if we have to # recalculate from scratch diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 00d66886ad..b96104ccae 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -45,19 +45,22 @@ from prometheus_client import Counter logger = logging.getLogger(__name__) persist_event_counter = Counter("synapse_storage_events_persisted_events", "") -event_counter = Counter("synapse_storage_events_persisted_events_sep", "", ["type", "origin_type", "origin_entity"]) +event_counter = Counter("synapse_storage_events_persisted_events_sep", "", + ["type", "origin_type", "origin_entity"]) # The number of times we are recalculating the current state state_delta_counter = Counter("synapse_storage_events_state_delta", "") # The number of times we are recalculating state when there is only a # single forward extremity -state_delta_single_event_counter = Counter("synapse_storage_events_state_delta_single_event", "") +state_delta_single_event_counter = Counter( + "synapse_storage_events_state_delta_single_event", "") # The number of times we are reculating state when we could have resonably # calculated the delta when we calculated the state for an event we were # persisting. -state_delta_reuse_delta_counter = Counter("synapse_storage_events_state_delta_reuse_delta", "") +state_delta_reuse_delta_counter = Counter( + "synapse_storage_events_state_delta_reuse_delta", "") def encode_json(json_object): diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index a964286d85..424fdcb036 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -28,17 +28,22 @@ block_counter = Counter("synapse_util_metrics_block_count", "", ["block_name"]) block_timer = Counter("synapse_util_metrics_block_time_seconds", "", ["block_name"]) -block_ru_utime = Counter("synapse_util_metrics_block_ru_utime_seconds", "", ["block_name"]) +block_ru_utime = Counter( + "synapse_util_metrics_block_ru_utime_seconds", "", ["block_name"]) -block_ru_stime = Counter("synapse_util_metrics_block_ru_stime_seconds", "", ["block_name"]) +block_ru_stime = Counter( + "synapse_util_metrics_block_ru_stime_seconds", "", ["block_name"]) -block_db_txn_count = Counter("synapse_util_metrics_block_db_txn_count", "", ["block_name"]) +block_db_txn_count = Counter( + "synapse_util_metrics_block_db_txn_count", "", ["block_name"]) # seconds spent waiting for db txns, excluding scheduling time, in this block -block_db_txn_duration = Counter("synapse_util_metrics_block_db_txn_duration_seconds", "", ["block_name"]) +block_db_txn_duration = Counter( + "synapse_util_metrics_block_db_txn_duration_seconds", "", ["block_name"]) # seconds spent waiting for a db connection, in this block -block_db_sched_duration = Counter("synapse_util_metrics_block_db_sched_duration_seconds", "", ["block_name"]) +block_db_sched_duration = Counter( + "synapse_util_metrics_block_db_sched_duration_seconds", "", ["block_name"]) def measure_func(name): From b6063631c3c5ea4c4d118cc9f9d7e2896949ee1d Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 22 May 2018 17:36:20 -0500 Subject: [PATCH 225/310] more cleanup --- synapse/metrics/__init__.py | 4 +--- synapse/replication/tcp/protocol.py | 9 ++++++--- synapse/replication/tcp/resource.py | 7 ++++--- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index bed37b5f56..75117915ff 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -21,7 +21,7 @@ import platform import attr from prometheus_client import Gauge, Histogram, Counter -from prometheus_client.core import GaugeMetricFamily, CounterMetricFamily, REGISTRY +from prometheus_client.core import GaugeMetricFamily, REGISTRY from twisted.internet import reactor @@ -97,8 +97,6 @@ gc_time = Histogram( class GCCounts(object): def collect(self): - gc_counts = gc.get_count() - cm = GaugeMetricFamily("python_gc_counts", "GC cycle counts", labels=["gen"]) for n, m in enumerate(gc.get_count()): cm.add_metric([str(n)], m) diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py index bdb110663c..5995e5d553 100644 --- a/synapse/replication/tcp/protocol.py +++ b/synapse/replication/tcp/protocol.py @@ -197,7 +197,8 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver): self.last_received_command = self.clock.time_msec() - self.inbound_commands_counter[cmd_name] = self.inbound_commands_counter[cmd_name] + 1 + self.inbound_commands_counter[cmd_name] = ( + self.inbound_commands_counter[cmd_name] + 1) cmd_cls = COMMAND_MAP[cmd_name] try: @@ -247,7 +248,8 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver): self._queue_command(cmd) return - self.outbound_commands_counter[cmd.NAME] = self.outbound_commands_counter[cmd.NAME] + 1 + self.outbound_commands_counter[cmd.NAME] = ( + self.outbound_commands_counter[cmd.NAME] + 1) string = "%s %s" % (cmd.NAME, cmd.to_line(),) if "\n" in string: raise Exception("Unexpected newline in command: %r", string) @@ -632,4 +634,5 @@ tcp_outbound_commands = LaterGauge( }) # number of updates received for each RDATA stream -inbound_rdata_count = Counter("synapse_replication_tcp_inbound_rdata_count", "", ["stream_name"]) +inbound_rdata_count = Counter("synapse_replication_tcp_inbound_rdata_count", "", + ["stream_name"]) diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index 1ca30bc31a..19987c06f0 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -28,12 +28,13 @@ import logging from prometheus_client import Counter -stream_updates_counter = Counter("synapse_replication_tcp_resource_stream_updates", "", ["stream_name"] -) +stream_updates_counter = Counter("synapse_replication_tcp_resource_stream_updates", + "", ["stream_name"]) user_sync_counter = Counter("synapse_replication_tcp_resource_user_sync", "") federation_ack_counter = Counter("synapse_replication_tcp_resource_federation_ack", "") remove_pusher_counter = Counter("synapse_replication_tcp_resource_remove_pusher", "") -invalidate_cache_counter = Counter("synapse_replication_tcp_resource_invalidate_cache", "") +invalidate_cache_counter = Counter("synapse_replication_tcp_resource_invalidate_cache", + "") user_ip_cache_counter = Counter("synapse_replication_tcp_resource_user_ip_cache", "") logger = logging.getLogger(__name__) From 0a4bca413425fee20aed2103d6886a7701ac6b09 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 23 May 2018 10:04:23 +0100 Subject: [PATCH 226/310] Use iter* methods for _filter_events_for_server --- synapse/handlers/federation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index ba3ede8024..c2dc1e2690 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -479,18 +479,18 @@ class FederationHandler(BaseHandler): # to get all state ids that we're interested in. event_map = yield self.store.get_events([ e_id - for key_to_eid in event_to_state_ids.values() - for key, e_id in key_to_eid.items() + for key_to_eid in event_to_state_ids.itervalues() + for key, e_id in key_to_eid.iteritems() if key[0] != EventTypes.Member or check_match(key[1]) ]) event_to_state = { e_id: { key: event_map[inner_e_id] - for key, inner_e_id in key_to_eid.items() + for key, inner_e_id in key_to_eid.iteritems() if inner_e_id in event_map } - for e_id, key_to_eid in event_to_state_ids.items() + for e_id, key_to_eid in event_to_state_ids.iteritems() } def redact_disallowed(event, state): From 5aaa3189d5a1ef0edc4ed09d7cc61e1a7452efed Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 23 May 2018 10:13:05 +0100 Subject: [PATCH 227/310] s/values/itervalues/ --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index c2dc1e2690..c15c8c1ed5 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -505,7 +505,7 @@ class FederationHandler(BaseHandler): # membership states for the requesting server to determine # if the server is either in the room or has been invited # into the room. - for ev in state.values(): + for ev in state.itervalues(): if ev.type != EventTypes.Member: continue try: From e7598b666ba6c91f69f312d367017caca8f44a8f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 11:14:23 +0100 Subject: [PATCH 228/310] Some docs about server notices --- docs/server_notices.md | 53 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 docs/server_notices.md diff --git a/docs/server_notices.md b/docs/server_notices.md new file mode 100644 index 0000000000..a37aed6792 --- /dev/null +++ b/docs/server_notices.md @@ -0,0 +1,53 @@ +Server Notices +============== + +'Server Notices' are a new feature introduced in Synapse 0.30. They provide a +channel whereby server administrators can send messages to users on the server. + +They are used as part of the communication of Privacy Polices (see +[privacy_policy.md]), however the intention is that they may also find a use +for features such as "Message of the day". + +This is a feature specific to Synapse, but it uses standard Matrix +communication mechanisms, so should work with any Matrix client. + +User experience +--------------- + +When the user is first sent a server notice, they will get an invitation to a +room (typically called 'Server Notices', though this is configurable in +`homeserver.yaml`). They will be **unable to reject** this invitation - +attempts to do so will receive an error. + +Once they accept the invitation, they will see the notice message in the room +history; it will appear to have come from the 'server notices user' (see +below). + +The user is prevented from sending any messages in this room by the power +levels. They also cannot leave it. + +Synapse configuration +--------------------- + +Server notices come from a specific user id on the server. Server +administrators are free to choose the user id - something like `server` is +suggested, meaning the notices will come from +`@server:`. Once the server notices user is configured, that +user id becomes a special, privileged user, so administrators should ensure +that **it is not already allocated**. + +In order to support server notices, it is necessary to add some configuration +to the `homeserver.yaml` file. In particular, you should add a `server_notices` +section, which should look like this: + +```yaml +server_notices: + system_mxid_localpart: server + system_mxid_display_name: "Server Notices" + room_name: "Server Notices" +``` + +The only compulsory setting is `system_mxid_localpart`, which defines the user +id of the server notices user, as above. `system_mxid_display_name` and +`room_name` define the displayname of the system notices user, and of +the notices room, respectively. From 3601a240aaec9af7c95c11821d851fd5fa03b0c8 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 23 May 2018 12:03:23 +0100 Subject: [PATCH 229/310] bump version and changelog --- CHANGES.rst | 36 ++++++++++++++++++++++++++++++++++++ synapse/__init__.py | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 851cb9b2d7..ee418abff3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,39 @@ +Changes in synapse v0.30.0-rc1 (2018-05-23) +========================================== + +This version includes support for GDPR consent collection. Users signal their +consent through responding to a private message from a bot named +@server:matrix.org For more information see XXXXXX + +GDPR Support: + +* ConsentResource to gather policy consent from users (PR #3213) +* Move RoomCreationHandler out of synapse.handlers.Handlers (PR #3225) +* Infrastructure for a server notices room (PR #3232) +* Send users a server notice about consent (PR #3236) +* Reject attempts to send event before privacy consent is given (PR #3257) +* Add a 'has_consented' template var to consent forms (PR #3262) +* Fix dependency on jinja2 (PR #3263) + +Features: +* Cohort analytics (PR #3163, #3241, #3251) +* Add lxml to docker image for web previews (PR #3239) Thanks to @ptman! +* Add in flight request metrics (PR #3252) + +Changes: + +* Remove unused `update_external_syncs` (PR #3233) +* Use stream rather depth ordering for push actions (PR #3212) +* Make purge_history operate on tokens (PR #3221) +* Don't support limitless pagination (PR #3265) + +Bug Fixes: + +* Fix logcontext resource usage tracking (PR #3258) +* Fix error in handling receipts (PR #3235) +* Stop the transaction cache caching failures (PR #3255) + + Changes in synapse v0.29.1 (2018-05-17) ========================================== Changes: diff --git a/synapse/__init__.py b/synapse/__init__.py index b11a1d3ea3..353387f154 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.29.1" +__version__ = "0.30.0-rc1" From 9e8ab0a4f44a3ec9e4b049f5571c14e333e8f0fa Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 23 May 2018 12:05:58 +0100 Subject: [PATCH 230/310] style --- CHANGES.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index ee418abff3..096fa4794a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,9 +1,9 @@ Changes in synapse v0.30.0-rc1 (2018-05-23) ========================================== -This version includes support for GDPR consent collection. Users signal their -consent through responding to a private message from a bot named -@server:matrix.org For more information see XXXXXX +This version includes support for Privacy Notice agreement collection. More +details live at docs/privacy_policy_templates/README.md + GDPR Support: @@ -16,6 +16,7 @@ GDPR Support: * Fix dependency on jinja2 (PR #3263) Features: + * Cohort analytics (PR #3163, #3241, #3251) * Add lxml to docker image for web previews (PR #3239) Thanks to @ptman! * Add in flight request metrics (PR #3252) From 833db2d92245a0b7cab9e16f9badc135346f0750 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 12:32:38 +0100 Subject: [PATCH 231/310] consent tracking docs --- docs/consent_tracking.md | 152 ++++++++++++++++++++++++ docs/privacy_policy_templates/README.md | 23 ---- docs/server_notices.md | 4 +- 3 files changed, 154 insertions(+), 25 deletions(-) create mode 100644 docs/consent_tracking.md delete mode 100644 docs/privacy_policy_templates/README.md diff --git a/docs/consent_tracking.md b/docs/consent_tracking.md new file mode 100644 index 0000000000..eb34749b55 --- /dev/null +++ b/docs/consent_tracking.md @@ -0,0 +1,152 @@ +Support in Synapse for tracking agreement to server terms and conditions +======================================================================== + +Synapse 0.30 introduces support for tracking whether users have agreed to the +terms and conditions set by the administrator of a server - and blocking access +to the server until they have. + +There are several parts to this functionality; each requires some specific +configuration in `homeserver.yaml` to be enabled. + +Note that various parts of the configuation and this document refer to the +"privacy policy": agreement with a privacy policy is one particular use of this +feature, but of course adminstrators can specify other terms and conditions +unrelated to "privacy" per se. + +Collecting policy agreement from a user +--------------------------------------- + +Synapse can be configured to serve the user a simple policy form with an +"accept" button. Clicking "Accept" records the user's acceptance in the +database and shows a success page. + +To enable this, first create templates for the policy and success pages. +These should be stored on the local filesystem. + +These templates use the [Jinja2](http://jinja.pocoo.org) templating language, +and the `privacy_policy_templates` subdirectory of this `docs` directory gives +examples of the sort of thing that can be done. + +Note that the templates must be stored under a name giving the language of the +template - currently this must always be `en` (for "English"); +internationalisation support is intended for the future. + +The template for the policy itself should be versioned - for example +`1.0.html`. The version of the policy which the user has agreed to is stored in +the database. + +Once the templates are in place, make the following changes to `homeserver.yaml`: + + 1. Add a `user_consent` section, which should look like: + + ```yaml + user_consent: + template_dir: privacy_policy_templates + version: 1.0 + ``` + + `template_dir` points to the directory containing the policy + templates. `version` defines the version of the policy which will be served + to the user. In the example above, Synapse will serve + `privacy_policy_templates/en/1.0.html`. + + + 2. Add a `form_secret` setting at the top level: + + + ```yaml + form_secret: "" + ``` + + This should be set to an arbitrary secret string (try `pwgen -y 30` to + generate suitable secrets). + + More on what this is used for below. + + 3. Add `consent` wherever the `client` resource is currently enabled in the + `listeners` configuration. For example: + + ``` + listeners: + - port: 8008 + resources: + - names: + - client + - consent + ``` + + +Finally, ensure that `jinja2` is installed. If you are using a virtualenv, this +should be a matter of `pip install Jinja2`. On debian, try `apt-get install +python-jinja2`. + +Once this is complete, and the server has been restarted, try visiting +`https:///_matrix/consent`. If configuration has been done correctly, +this should give an error "Missing string query parameter 'u'". It is now +possible to manually construct URIs where users can give their consent. + +Constructing the consent URI +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It may be useful to manually construct the "consent URI" for a given user - for +instance, in order to send them an email asking them to consent. To do this, +take the base `https:///_matrix/consent` URL and add the following +query parameters: + + * `u`: the user id of the user. This can either be a full MXID + (`@user:server.com`) or just the localpart (`user`). + + * `h`: hex-encoded HMAC-SHA256 of `u` using the `form_secret` as a key. It is + possible to calculate this on the commandline with something like: + + ```bash + echo -n '' | openssl sha256 -hmac '' + ``` + + This should result in a URI which looks something like: + `https:///_matrix/consent?u=&h=68a152465a4d...`. + + +Sending users a server notice asking them to agree to the policy +---------------------------------------------------------------- + +It is possible to configure Synapse to send a [server +notice](server_notices.md) to anybody who has not yet agreed to the current +version of the policy. To do so: + + * ensure that the consent resource is configured, as in the previous section + + * ensure that server notices are configured, as in [server_notices.md]. + + * Add `server_notice_content` under `user_consent` in `homeserver.yaml`. For + example: + + ```yaml + user_consent: + server_notice_content: + msgtype: m.text + body: >- + Please give your consent to the privacy policy at %(consent_uri)s. + ``` + + Synapse automatically replaces the placeholder `%(consent_uri)s` with the + consent uri for that user. + +Blocking users from using the server until they agree to the policy +------------------------------------------------------------------- + +Synapse can be configured to block any attempts to join rooms or send messages +until the user has given their agreement to the policy. (Joining the server +notices room is exempted from this). + +To enable this, add `block_events_error` under `user_consent`. For example: + +``` +user_consent: + block_events_error: >- + You can't send any messages until you consent to the privacy policy at + %(consent_uri)s. +``` + +Synapse automatically replaces the placeholder `%(consent_uri)s` with the +consent uri for that user. diff --git a/docs/privacy_policy_templates/README.md b/docs/privacy_policy_templates/README.md deleted file mode 100644 index a3e6fc0986..0000000000 --- a/docs/privacy_policy_templates/README.md +++ /dev/null @@ -1,23 +0,0 @@ -If enabling the 'consent' resource in synapse, you will need some templates -for the HTML to be served to the user. This directory contains very simple -examples of the sort of thing that can be done. - -You'll need to add this sort of thing to your homeserver.yaml: - -``` -form_secret: - -user_consent: - template_dir: docs/privacy_policy_templates - version: 1.0 -``` - -You should then be able to enable the `consent` resource under a `listener` -entry. For example: - -``` -listeners: - - port: 8008 - resources: - - names: [client, consent] -``` diff --git a/docs/server_notices.md b/docs/server_notices.md index a37aed6792..5d7857d15c 100644 --- a/docs/server_notices.md +++ b/docs/server_notices.md @@ -4,8 +4,8 @@ Server Notices 'Server Notices' are a new feature introduced in Synapse 0.30. They provide a channel whereby server administrators can send messages to users on the server. -They are used as part of the communication of Privacy Polices (see -[privacy_policy.md]), however the intention is that they may also find a use +They are used as part of the communication of the server polices (see +[consent_tracking.md]), however the intention is that they may also find a use for features such as "Message of the day". This is a feature specific to Synapse, but it uses standard Matrix From 2574ea3dc83c58d1cb596606ac760d91022f336f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 23 May 2018 12:34:34 +0100 Subject: [PATCH 232/310] server_notices.md: fix link --- docs/server_notices.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/server_notices.md b/docs/server_notices.md index 5d7857d15c..22bca332d5 100644 --- a/docs/server_notices.md +++ b/docs/server_notices.md @@ -5,8 +5,8 @@ Server Notices channel whereby server administrators can send messages to users on the server. They are used as part of the communication of the server polices (see -[consent_tracking.md]), however the intention is that they may also find a use -for features such as "Message of the day". +[consent_tracking.md](consent_tracking.md)), however the intention is that +they may also find a use for features such as "Message of the day". This is a feature specific to Synapse, but it uses standard Matrix communication mechanisms, so should work with any Matrix client. From 563606b8f2216f6a8ba64c4b55bf389f67c99e9e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 23 May 2018 12:37:39 +0100 Subject: [PATCH 233/310] consent_tracking: formatting etc --- docs/consent_tracking.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/consent_tracking.md b/docs/consent_tracking.md index eb34749b55..61e41468d5 100644 --- a/docs/consent_tracking.md +++ b/docs/consent_tracking.md @@ -31,9 +31,9 @@ Note that the templates must be stored under a name giving the language of the template - currently this must always be `en` (for "English"); internationalisation support is intended for the future. -The template for the policy itself should be versioned - for example -`1.0.html`. The version of the policy which the user has agreed to is stored in -the database. +The template for the policy itself should be versioned and named according to +the version: for example `1.0.html`. The version of the policy which the user +has agreed to is stored in the database. Once the templates are in place, make the following changes to `homeserver.yaml`: @@ -66,7 +66,7 @@ Once the templates are in place, make the following changes to `homeserver.yaml` 3. Add `consent` wherever the `client` resource is currently enabled in the `listeners` configuration. For example: - ``` + ```yaml listeners: - port: 8008 resources: @@ -85,8 +85,7 @@ Once this is complete, and the server has been restarted, try visiting this should give an error "Missing string query parameter 'u'". It is now possible to manually construct URIs where users can give their consent. -Constructing the consent URI -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +### Constructing the consent URI It may be useful to manually construct the "consent URI" for a given user - for instance, in order to send them an email asking them to consent. To do this, @@ -116,7 +115,7 @@ version of the policy. To do so: * ensure that the consent resource is configured, as in the previous section - * ensure that server notices are configured, as in [server_notices.md]. + * ensure that server notices are configured, as in [server_notices.md](server_notices.md). * Add `server_notice_content` under `user_consent` in `homeserver.yaml`. For example: @@ -141,7 +140,7 @@ notices room is exempted from this). To enable this, add `block_events_error` under `user_consent`. For example: -``` +```yaml user_consent: block_events_error: >- You can't send any messages until you consent to the privacy policy at From 5ad1149f384e1a826ad73a791896d63c910d874c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 13:45:17 +0100 Subject: [PATCH 234/310] Notes on the manhole --- docs/manhole.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 docs/manhole.md diff --git a/docs/manhole.md b/docs/manhole.md new file mode 100644 index 0000000000..7375f5ad46 --- /dev/null +++ b/docs/manhole.md @@ -0,0 +1,43 @@ +Using the synapse manhole +========================= + +The "manhole" allows server administrators to access a Python shell on a running +Synapse installation. This is a very powerful mechanism for administration and +debugging. + +To enable it, first uncomment the `manhole` listener configuration in +`homeserver.yaml`: + +```yaml +listeners: + - port: 9000 + bind_addresses: ['::1', '127.0.0.1'] + type: manhole +``` + +(`bind_addresses` in the above is important: it ensures that access to the +manhole is only possible for local users). + +Note that this will give administrative access to synapse to **all users** with +shell access to the server. It should therefore **not** be enabled in +environments where untrusted users have shell access. + +Then restart synapse, and point an ssh client at port 9000 on localhost, using +the username `matrix`: + +```bash +ssh -p9000 matrix@localhost +``` + +The password is `rabbithole`. + +This gives a Python REPL in which `hs` gives access to the +`synapse.server.HomeServer` object - which in turn gives access to many other +parts of the process. + +As a simple example, retrieving an event from the database: + +``` +>>> hs.get_datastore().get_event('$1416420717069yeQaw:matrix.org') +> +``` From 052d08a6a579f4f9a98a76a3cb39c05d8d5aaa90 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 13:55:39 +0100 Subject: [PATCH 235/310] Using the manhole to send server notices --- docs/server_notices.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/server_notices.md b/docs/server_notices.md index 22bca332d5..a96a5b88a3 100644 --- a/docs/server_notices.md +++ b/docs/server_notices.md @@ -51,3 +51,16 @@ The only compulsory setting is `system_mxid_localpart`, which defines the user id of the server notices user, as above. `system_mxid_display_name` and `room_name` define the displayname of the system notices user, and of the notices room, respectively. + +Sending notices +--------------- + +As of the current version of synapse, there is no convenient interface for +sending notices (other than the automated ones sent as part of consent +tracking). + +In the meantime, it is possible to test this feature using the manhole. Having gone into the manhole as described in [manhole.md](manhole.md), a notice can be sent with something like: + +``` +>>> hs.get_server_notices_manager().send_notice('@user:server.com', {'msgtype':'m.text', 'body':'foo'}) +``` From 1cbb8e5a33326441ec311ce212bb62fcd6704669 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 13:58:28 +0100 Subject: [PATCH 236/310] fix wrapping --- docs/server_notices.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/server_notices.md b/docs/server_notices.md index a96a5b88a3..9896881389 100644 --- a/docs/server_notices.md +++ b/docs/server_notices.md @@ -59,7 +59,9 @@ As of the current version of synapse, there is no convenient interface for sending notices (other than the automated ones sent as part of consent tracking). -In the meantime, it is possible to test this feature using the manhole. Having gone into the manhole as described in [manhole.md](manhole.md), a notice can be sent with something like: +In the meantime, it is possible to test this feature using the manhole. Having +gone into the manhole as described in [manhole.md](manhole.md), a notice can be +sent with something like: ``` >>> hs.get_server_notices_manager().send_notice('@user:server.com', {'msgtype':'m.text', 'body':'foo'}) From 321f02d26347fb56e1eafbba60d7faef95ae9c3f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 14:30:47 +0100 Subject: [PATCH 237/310] Block attempts to send server notices to remote users --- synapse/server_notices/server_notices_manager.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index f535b9c9da..9f76c75546 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -35,6 +35,7 @@ class ServerNoticesManager(object): self._config = hs.config self._room_creation_handler = hs.get_room_creation_handler() self._event_creation_handler = hs.get_event_creation_handler() + self._is_mine_id = hs.is_mine_id def is_enabled(self): """Checks if server notices are enabled on this server. @@ -89,6 +90,9 @@ class ServerNoticesManager(object): if not self.is_enabled(): raise Exception("Server notices not enabled") + assert self._is_mine_id(user_id), \ + "Cannot send server notices to remote users" + rooms = yield self._store.get_rooms_for_user_where_membership_is( user_id, [Membership.INVITE, Membership.JOIN], ) From cd8ab9a0d8d3e0ee305ab56788f76459596075f9 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 14:43:09 +0100 Subject: [PATCH 238/310] mention public_baseurl --- docs/consent_tracking.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/consent_tracking.md b/docs/consent_tracking.md index 61e41468d5..66beb9263b 100644 --- a/docs/consent_tracking.md +++ b/docs/consent_tracking.md @@ -131,6 +131,11 @@ version of the policy. To do so: Synapse automatically replaces the placeholder `%(consent_uri)s` with the consent uri for that user. + * ensure that `public_baseurl` is set in `homeserver.yaml`, and gives the base + URI that clients use to connect to the server. (It is used to construct + `consent_uri` in the server notice.) + + Blocking users from using the server until they agree to the policy ------------------------------------------------------------------- @@ -149,3 +154,7 @@ user_consent: Synapse automatically replaces the placeholder `%(consent_uri)s` with the consent uri for that user. + +ensure that `public_baseurl` is set in `homeserver.yaml`, and gives the base +URI that clients use to connect to the server. (It is used to construct +`consent_uri` in the error.) From 82191b08f6a45c693551983a48b49a9fc30184c4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 15:24:31 +0100 Subject: [PATCH 239/310] Support for putting %(consent_uri)s in messages Make it possible to put the URI in the error message and the server notice that get sent by the server --- synapse/config/consent_config.py | 11 +++-- synapse/handlers/message.py | 5 ++- .../server_notices/consent_server_notices.py | 41 ++++++++++++++++++- 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py index 44c4711e6c..ddcd305f4c 100644 --- a/synapse/config/consent_config.py +++ b/synapse/config/consent_config.py @@ -43,10 +43,13 @@ DEFAULT_CONFIG = """\ # version: 1.0 # server_notice_content: # msgtype: m.text -# body: | -# Pls do consent kthx -# block_events_error: | -# You can't send any messages until you consent to the privacy policy. +# body: >- +# To continue using this homeserver you must review and agree to the +# terms and conditions at %(consent_uri)s +# block_events_error: >- +# To continue using this homeserver you must review and agree to the +# terms and conditions at %(consent_uri)s +# """ diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index c3adbc6c95..0886607023 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -575,8 +575,11 @@ class EventCreationHandler(object): return consent_uri = self._consent_uri_builder.build_user_consent_uri(user_id) + msg = self.config.block_events_without_consent_error % { + 'consent_uri': consent_uri, + } raise ConsentNotGivenError( - msg=self.config.block_events_without_consent_error, + msg=msg, consent_uri=consent_uri, ) diff --git a/synapse/server_notices/consent_server_notices.py b/synapse/server_notices/consent_server_notices.py index 440f6b1cd4..a709802856 100644 --- a/synapse/server_notices/consent_server_notices.py +++ b/synapse/server_notices/consent_server_notices.py @@ -14,10 +14,13 @@ # limitations under the License. import logging +from six import (iteritems, string_types) from twisted.internet import defer from synapse.api.errors import SynapseError +from synapse.api.urls import ConsentURIBuilder from synapse.config import ConfigError +from synapse.types import get_localpart_from_id logger = logging.getLogger(__name__) @@ -52,6 +55,8 @@ class ConsentServerNotices(object): "key.", ) + self._consent_uri_builder = ConsentURIBuilder(hs.config) + @defer.inlineCallbacks def maybe_send_server_notice_to_user(self, user_id): """Check if we need to send a notice to this user, and does so if so @@ -81,10 +86,18 @@ class ConsentServerNotices(object): # we've already sent a notice to the user return - # need to send a message + # need to send a message. try: + consent_uri = self._consent_uri_builder.build_user_consent_uri( + get_localpart_from_id(user_id), + ) + content = copy_with_str_subst( + self._server_notice_content, { + 'consent_uri': consent_uri, + }, + ) yield self._server_notices_manager.send_notice( - user_id, self._server_notice_content, + user_id, content, ) yield self._store.user_set_consent_server_notice_sent( user_id, self._current_consent_version, @@ -93,3 +106,27 @@ class ConsentServerNotices(object): logger.error("Error sending server notice about user consent: %s", e) finally: self._users_in_progress.remove(user_id) + + +def copy_with_str_subst(x, substitutions): + """Deep-copy a structure, carrying out string substitions on any strings + + Args: + x (object): structure to be copied + substitutions (object): substitutions to be made - passed into the + string '%' operator + + Returns: + copy of x + """ + if isinstance(x, string_types): + return x % substitutions + if isinstance(x, dict): + return { + k: copy_with_str_subst(v, substitutions) for (k, v) in iteritems(x) + } + if isinstance(x, (list, tuple)): + return [copy_with_str_subst(y) for y in x] + + # assume it's uninterested and can be shallow-copied. + return x From 3ff6f50eac86b063706f68387d3cf3e96a79a541 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 15:28:23 +0100 Subject: [PATCH 240/310] Use the localpart in the consent uri ... because it's shorter. --- synapse/handlers/message.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index c3adbc6c95..7936a60af4 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -574,7 +574,9 @@ class EventCreationHandler(object): if u["consent_version"] == self.config.user_consent_version: return - consent_uri = self._consent_uri_builder.build_user_consent_uri(user_id) + consent_uri = self._consent_uri_builder.build_user_consent_uri( + requester.user.localpart, + ) raise ConsentNotGivenError( msg=self.config.block_events_without_consent_error, consent_uri=consent_uri, From dda40fb55d0aa7bde6387e5e32ffed872fddea4e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 15:29:58 +0100 Subject: [PATCH 241/310] fix typo --- synapse/server_notices/server_notices_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index 9f76c75546..7e4055c84a 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -56,7 +56,7 @@ class ServerNoticesManager(object): event_content (dict): content of event to send Returns: - Deferrred[None] + Deferred[None] """ room_id = yield self.get_notice_room_for_user(user_id) From 2df8c3139ab7be10af4aaa750bbdc4ad39209583 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 15:39:52 +0100 Subject: [PATCH 242/310] minor post-review tweaks --- docs/consent_tracking.md | 6 +++--- docs/server_notices.md | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/consent_tracking.md b/docs/consent_tracking.md index 66beb9263b..9dc0093888 100644 --- a/docs/consent_tracking.md +++ b/docs/consent_tracking.md @@ -81,9 +81,9 @@ should be a matter of `pip install Jinja2`. On debian, try `apt-get install python-jinja2`. Once this is complete, and the server has been restarted, try visiting -`https:///_matrix/consent`. If configuration has been done correctly, -this should give an error "Missing string query parameter 'u'". It is now -possible to manually construct URIs where users can give their consent. +`https:///_matrix/consent`. If correctly configured, this should give +an error "Missing string query parameter 'u'". It is now possible to manually +construct URIs where users can give their consent. ### Constructing the consent URI diff --git a/docs/server_notices.md b/docs/server_notices.md index 9896881389..8e18e3d95d 100644 --- a/docs/server_notices.md +++ b/docs/server_notices.md @@ -4,7 +4,7 @@ Server Notices 'Server Notices' are a new feature introduced in Synapse 0.30. They provide a channel whereby server administrators can send messages to users on the server. -They are used as part of the communication of the server polices (see +They are used as part of communication of the server polices(see [consent_tracking.md](consent_tracking.md)), however the intention is that they may also find a use for features such as "Message of the day". @@ -32,7 +32,7 @@ Synapse configuration Server notices come from a specific user id on the server. Server administrators are free to choose the user id - something like `server` is suggested, meaning the notices will come from -`@server:`. Once the server notices user is configured, that +`@server:`. Once the Server Notices user is configured, that user id becomes a special, privileged user, so administrators should ensure that **it is not already allocated**. From e206b2c9ace8a202bf47ec419581e19f0baa39fb Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 23 May 2018 15:57:10 +0100 Subject: [PATCH 243/310] consent_tracking.md: clarify link --- docs/consent_tracking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/consent_tracking.md b/docs/consent_tracking.md index 9dc0093888..064eae82f7 100644 --- a/docs/consent_tracking.md +++ b/docs/consent_tracking.md @@ -24,7 +24,7 @@ To enable this, first create templates for the policy and success pages. These should be stored on the local filesystem. These templates use the [Jinja2](http://jinja.pocoo.org) templating language, -and the `privacy_policy_templates` subdirectory of this `docs` directory gives +and [docs/privacy_policy_templates](privacy_policy_templates) gives examples of the sort of thing that can be done. Note that the templates must be stored under a name giving the language of the From 9bf4b2bda343dcbdba7b0e9d752bc560f8e344fd Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 17:43:30 +0100 Subject: [PATCH 244/310] Allow overriding the server_notices user's avatar probably should have done this in the first place, like @turt2live suggested. --- docs/server_notices.md | 9 ++++++--- synapse/config/server_notices_config.py | 15 ++++++++++++--- .../server_notices/server_notices_manager.py | 17 ++++++++++++++--- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/docs/server_notices.md b/docs/server_notices.md index 8e18e3d95d..221553b24d 100644 --- a/docs/server_notices.md +++ b/docs/server_notices.md @@ -44,13 +44,16 @@ section, which should look like this: server_notices: system_mxid_localpart: server system_mxid_display_name: "Server Notices" + system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ" room_name: "Server Notices" ``` The only compulsory setting is `system_mxid_localpart`, which defines the user -id of the server notices user, as above. `system_mxid_display_name` and -`room_name` define the displayname of the system notices user, and of -the notices room, respectively. +id of the Server Notices user, as above. `room_name` defines the name of the +room which will be created. + +`system_mxid_display_name` and `system_mxid_avatar_url` can be used to set the +displayname and avatar of the Server Notices user. Sending notices --------------- diff --git a/synapse/config/server_notices_config.py b/synapse/config/server_notices_config.py index ccef8d2ec5..be1d1f762c 100644 --- a/synapse/config/server_notices_config.py +++ b/synapse/config/server_notices_config.py @@ -26,12 +26,13 @@ DEFAULT_CONFIG = """\ # setting, which defines the id of the user which will be used to send the # notices. # -# It's also possible to override the room name, or the display name of the -# "notices" user. +# It's also possible to override the room name, the display name of the +# "notices" user, and the avatar for the user. # # server_notices: # system_mxid_localpart: notices # system_mxid_display_name: "Server Notices" +# system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ" # room_name: "Server Notices" """ @@ -48,6 +49,10 @@ class ServerNoticesConfig(Config): The display name to use for the server notices user. None if server notices are not enabled. + server_notices_mxid_avatar_url (str|None): + The display name to use for the server notices user. + None if server notices are not enabled. + server_notices_room_name (str|None): The name to use for the server notices room. None if server notices are not enabled. @@ -56,6 +61,7 @@ class ServerNoticesConfig(Config): super(ServerNoticesConfig, self).__init__() self.server_notices_mxid = None self.server_notices_mxid_display_name = None + self.server_notices_mxid_avatar_url = None self.server_notices_room_name = None def read_config(self, config): @@ -68,7 +74,10 @@ class ServerNoticesConfig(Config): mxid_localpart, self.server_name, ).to_string() self.server_notices_mxid_display_name = c.get( - 'system_mxid_display_name', 'Server Notices', + 'system_mxid_display_name', None, + ) + self.server_notices_mxid_avatar_url = c.get( + 'system_mxid_avatar_url', None, ) # todo: i18n self.server_notices_room_name = c.get('room_name', "Server Notices") diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index 7e4055c84a..a26deace53 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -113,6 +113,19 @@ class ServerNoticesManager(object): # apparently no existing notice room: create a new one logger.info("Creating server notices room for %s", user_id) + # see if we want to override the profile info for the server user. + # note that if we want to override either the display name or the + # avatar, we have to use both. + join_profile = None + if ( + self._config.server_notices_mxid_display_name is not None or + self._config.server_notices_mxid_avatar_url is not None + ): + join_profile = { + "displayname": self._config.server_notices_mxid_display_name, + "avatar_url": self._config.server_notices_mxid_avatar_url, + } + requester = create_requester(system_mxid) info = yield self._room_creation_handler.create_room( requester, @@ -125,9 +138,7 @@ class ServerNoticesManager(object): "invite": (user_id,) }, ratelimit=False, - creator_join_profile={ - "displayname": self._config.server_notices_mxid_display_name, - }, + creator_join_profile=join_profile, ) room_id = info['room_id'] From e987079037eb89a3c5c04fe64d843a6f9e6c1718 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 23 May 2018 13:03:51 -0500 Subject: [PATCH 245/310] fixes --- synapse/federation/transaction_queue.py | 8 ++++---- synapse/http/request_metrics.py | 14 +++++++------- synapse/push/bulk_push_rule_evaluator.py | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 53442688c8..69312ec233 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -41,7 +41,7 @@ import logging logger = logging.getLogger(__name__) sent_pdus_destination_dist = Counter( - "synapse_federation_client_sent_pdu_destinations", "" + "synapse_federation_transaction_queue_sent_pdu_destinations", "" ) @@ -70,7 +70,7 @@ class TransactionQueue(object): self.pending_transactions = {} LaterGauge( - "synapse_federation_client_pending_destinations", + "synapse_federation_transaction_queue_pending_destinations", "", [], lambda: len(self.pending_transactions), @@ -97,13 +97,13 @@ class TransactionQueue(object): self.pending_edus_keyed_by_dest = edus_keyed = {} LaterGauge( - "synapse_federation_client_pending_pdus", + "synapse_federation_transaction_queue_pending_pdus", "", [], lambda: sum(map(len, pdus.values())), ) LaterGauge( - "synapse_federation_client_pending_edus", + "synapse_federation_transaction_queue_pending_edus", "", [], lambda: ( diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py index 7f11b5c5a4..e7df494333 100644 --- a/synapse/http/request_metrics.py +++ b/synapse/http/request_metrics.py @@ -63,46 +63,46 @@ response_db_txn_duration = Counter( # seconds spent waiting for a db connection, when processing this request response_db_sched_duration = Counter( - "synapse_http_request_response_db_sched_duration_seconds", + "synapse_http_server_response_db_sched_duration_seconds", "", ["method", "servlet", "tag"], ) # size in bytes of the response written response_size = Counter( - "synapse_http_request_response_size", "", ["method", "servlet", "tag"] + "synapse_http_server_response_size", "", ["method", "servlet", "tag"] ) # In flight metrics are incremented while the requests are in flight, rather # than when the response was written. in_flight_requests_ru_utime = Counter( - "synapse_http_request_in_flight_requests_ru_utime_seconds", + "synapse_http_server_in_flight_requests_ru_utime_seconds", "", ["method", "servlet"], ) in_flight_requests_ru_stime = Counter( - "synapse_http_request_in_flight_requests_ru_stime_seconds", + "synapse_http_server_in_flight_requests_ru_stime_seconds", "", ["method", "servlet"], ) in_flight_requests_db_txn_count = Counter( - "synapse_http_request_in_flight_requests_db_txn_count", "", ["method", "servlet"] + "synapse_http_server_in_flight_requests_db_txn_count", "", ["method", "servlet"] ) # seconds spent waiting for db txns, excluding scheduling time, when processing # this request in_flight_requests_db_txn_duration = Counter( - "synapse_http_request_in_flight_requests_db_txn_duration_seconds", + "synapse_http_server_in_flight_requests_db_txn_duration_seconds", "", ["method", "servlet"], ) # seconds spent waiting for a db connection, when processing this request in_flight_requests_db_sched_duration = Counter( - "synapse_http_request_in_flight_requests_db_sched_duration_seconds", + "synapse_http_server_in_flight_requests_db_sched_duration_seconds", "", ["method", "servlet"], ) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index b0053e7f3f..da1cde417f 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -37,9 +37,9 @@ rules_by_room = {} push_rules_invalidation_counter = Counter( - "synapse_push_bulk_push_role_evaluator_push_rules_invalidation_counter", "") + "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter", "") push_rules_state_size_counter = Counter( - "synapse_push_bulk_push_role_evaluator_push_rules_state_size_counter", "") + "synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter", "") # Measures whether we use the fast path of using state deltas, or if we have to # recalculate from scratch From 472a5ec4e208967aaf1368ada321aa54c3cae493 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 23 May 2018 13:03:56 -0500 Subject: [PATCH 246/310] add back CPU metrics --- synapse/metrics/__init__.py | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index 75117915ff..64ad762ff1 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -17,6 +17,7 @@ import logging import functools import time import gc +import os import platform import attr @@ -81,6 +82,38 @@ class LaterGauge(object): all_gauges[self.name] = self +# +# Detailed CPU metrics +# + +class CPUMetrics(object): + + def __init__(self): + ticks_per_sec = 100 + try: + # Try and get the system config + ticks_per_sec = os.sysconf('SC_CLK_TCK') + except (ValueError, TypeError, AttributeError): + pass + + self.ticks_per_sec = ticks_per_sec + + def collect(self): + + with open("/proc/self/stat") as s: + line = s.read() + raw_stats = line.split(") ", 1)[1].split(" ") + + user = GaugeMetricFamily("process_cpu_user_seconds_total", "") + user.add_metric([], float(raw_stats[11]) / self.ticks_per_sec) + yield user + + sys = GaugeMetricFamily("process_cpu_system_seconds_total", "") + sys.add_metric([], float(raw_stats[12]) / self.ticks_per_sec) + yield sys + +REGISTRY.register(CPUMetrics()) + # # Python GC metrics # @@ -90,7 +123,8 @@ gc_time = Histogram( "python_gc_time", "Time taken to GC (ms)", ["gen"], - buckets=[1, 2, 5, 10, 25, 50, 100, 250, 500, 1000], + buckets=[2.5, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 7500, 15000, + 30000, 45000, 60000], ) From 389dac2c153f440d7ad6e9aee137ef40970b1394 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 23 May 2018 13:08:59 -0500 Subject: [PATCH 247/310] pepeightttt --- synapse/metrics/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index 64ad762ff1..e33ed6c9be 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -112,6 +112,7 @@ class CPUMetrics(object): sys.add_metric([], float(raw_stats[12]) / self.ticks_per_sec) yield sys + REGISTRY.register(CPUMetrics()) # From 91d95a1d8e4005ebd4805418d331a6ea8f73fdb6 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Thu, 24 May 2018 14:05:07 +0100 Subject: [PATCH 248/310] bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 353387f154..5bada5e290 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.30.0-rc1" +__version__ = "0.30.0" From bdd2ed5acf7ea34092c9aa7049232ea5e8f29874 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Thu, 24 May 2018 15:02:03 +0100 Subject: [PATCH 249/310] update for v0.30.0 --- CHANGES.rst | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 096fa4794a..ffadd83e4e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,11 +1,30 @@ +Changes in synapse v0.30.0 (2018-05-24) +========================================== + +'Server Notices' are a new feature introduced in Synapse 0.30. They provide a +channel whereby server administrators can send messages to users on the server. + +They are used as part of communication of the server policies (see `docs/consent_tracking.md`), +however the intention is that they may also find a use for features such +as "Message of the day". + +This feature is specific to Synapse, but uses standard Matrix communication mechanisms, +so should work with any Matrix client. For more details see ``docs/server_notices.md`` + +Further Server Notices/Consent Tracking Support: + +* Allow overriding the server_notices user's avatar (#3273) +* Use the localpart in the consent uri (#3272) +* Support for putting %(consent_uri)s in messages (#3271) +* Block attempts to send server notices to remote users (#3270) +* Docs on consent bits (#3268) + + + Changes in synapse v0.30.0-rc1 (2018-05-23) ========================================== -This version includes support for Privacy Notice agreement collection. More -details live at docs/privacy_policy_templates/README.md - - -GDPR Support: +Server Notices/Consent Tracking Support: * ConsentResource to gather policy consent from users (PR #3213) * Move RoomCreationHandler out of synapse.handlers.Handlers (PR #3225) From 6f67163c6358554bfd9568e98e0ac2225436b1c1 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Thu, 24 May 2018 15:04:41 +0100 Subject: [PATCH 250/310] Update CHANGES.rst --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index ffadd83e4e..6e7f108f8e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,7 @@ Changes in synapse v0.30.0 (2018-05-24) 'Server Notices' are a new feature introduced in Synapse 0.30. They provide a channel whereby server administrators can send messages to users on the server. -They are used as part of communication of the server policies (see `docs/consent_tracking.md`), +They are used as part of communication of the server policies (see ``docs/consent_tracking.md``), however the intention is that they may also find a use for features such as "Message of the day". From 14ca67867422d333a933b3bb0a4fd86da1656a98 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Thu, 24 May 2018 15:54:02 +0100 Subject: [PATCH 251/310] Update CHANGES.rst --- CHANGES.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6e7f108f8e..0569b581db 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -13,11 +13,11 @@ so should work with any Matrix client. For more details see ``docs/server_notice Further Server Notices/Consent Tracking Support: -* Allow overriding the server_notices user's avatar (#3273) -* Use the localpart in the consent uri (#3272) -* Support for putting %(consent_uri)s in messages (#3271) -* Block attempts to send server notices to remote users (#3270) -* Docs on consent bits (#3268) +* Allow overriding the server_notices user's avatar (PR #3273) +* Use the localpart in the consent uri (PR #3272) +* Support for putting %(consent_uri)s in messages (PR #3271) +* Block attempts to send server notices to remote users (PR #3270) +* Docs on consent bits (PR #3268) From 7a1af504d70e168e87f1e15ecda90179dd75ebc8 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 24 May 2018 15:59:58 +0100 Subject: [PATCH 252/310] Remove users from user directory on deactivate --- synapse/handlers/deactivate_account.py | 4 ++++ synapse/handlers/user_directory.py | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index d58ea6c650..c5e92f6214 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -30,6 +30,7 @@ class DeactivateAccountHandler(BaseHandler): self._auth_handler = hs.get_auth_handler() self._device_handler = hs.get_device_handler() self._room_member_handler = hs.get_room_member_handler() + self.user_directory_handler = hs.get_user_directory_handler() # Flag that indicates whether the process to part users from rooms is running self._user_parter_running = False @@ -65,6 +66,9 @@ class DeactivateAccountHandler(BaseHandler): # removal from all the rooms they're a member of) yield self.store.add_user_pending_deactivation(user_id) + # delete from user directory + yield self.user_directory_handler.handle_user_deactivated(user_id) + # Now start the process that goes through that list and # parts users from rooms (if it isn't already running) self._start_user_parting() diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 714f0195c8..2f73d664f9 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -122,6 +122,13 @@ class UserDirectoryHandler(object): user_id, profile.display_name, profile.avatar_url, None, ) + @defer.inlineCallbacks + def handle_user_deactivated(self, user_id): + """Called when a user ID is deactivated + """ + yield self.store.remove_from_user_dir(user_id) + yield self.store.remove_from_user_in_public_room(user_id) + @defer.inlineCallbacks def _unsafe_process(self): # If self.pos is None then means we haven't fetched it from DB From 095292304f04c0984813a261a7f6042b35cbd2dd Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Tue, 15 May 2018 17:52:41 +0200 Subject: [PATCH 253/310] Py3 storage/_base.py Signed-off-by: Adrian Tschira --- synapse/storage/_base.py | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 2262776ab2..51cf7b4c6c 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -27,6 +27,8 @@ import sys import time import threading +from six import itervalues, iterkeys, iteritems +from six.moves import intern, range logger = logging.getLogger(__name__) @@ -137,7 +139,7 @@ class PerformanceCounters(object): def interval(self, interval_duration, limit=3): counters = [] - for name, (count, cum_time) in self.current_counters.iteritems(): + for name, (count, cum_time) in iteritems(self.current_counters): prev_count, prev_time = self.previous_counters.get(name, (0, 0)) counters.append(( (cum_time - prev_time) / interval_duration, @@ -543,7 +545,7 @@ class SQLBaseStore(object): ", ".join("%s = ?" % (k,) for k in values), " AND ".join("%s = ?" % (k,) for k in keyvalues) ) - sqlargs = values.values() + keyvalues.values() + sqlargs = list(values.values()) + list(keyvalues.values()) txn.execute(sql, sqlargs) if txn.rowcount > 0: @@ -561,7 +563,7 @@ class SQLBaseStore(object): ", ".join(k for k in allvalues), ", ".join("?" for _ in allvalues) ) - txn.execute(sql, allvalues.values()) + txn.execute(sql, list(allvalues.values())) # successfully inserted return True @@ -629,8 +631,8 @@ class SQLBaseStore(object): } if keyvalues: - sql += " WHERE %s" % " AND ".join("%s = ?" % k for k in keyvalues.iterkeys()) - txn.execute(sql, keyvalues.values()) + sql += " WHERE %s" % " AND ".join("%s = ?" % k for k in iterkeys(keyvalues)) + txn.execute(sql, list(keyvalues.values())) else: txn.execute(sql) @@ -694,7 +696,7 @@ class SQLBaseStore(object): table, " AND ".join("%s = ?" % (k, ) for k in keyvalues) ) - txn.execute(sql, keyvalues.values()) + txn.execute(sql, list(keyvalues.values())) else: sql = "SELECT %s FROM %s" % ( ", ".join(retcols), @@ -725,9 +727,12 @@ class SQLBaseStore(object): if not iterable: defer.returnValue(results) + # iterables can not be sliced, so convert it to a list first + it_list = list(iterable) + chunks = [ - iterable[i:i + batch_size] - for i in xrange(0, len(iterable), batch_size) + it_list[i:i + batch_size] + for i in range(0, len(it_list), batch_size) ] for chunk in chunks: rows = yield self.runInteraction( @@ -767,7 +772,7 @@ class SQLBaseStore(object): ) values.extend(iterable) - for key, value in keyvalues.iteritems(): + for key, value in iteritems(keyvalues): clauses.append("%s = ?" % (key,)) values.append(value) @@ -790,7 +795,7 @@ class SQLBaseStore(object): @staticmethod def _simple_update_txn(txn, table, keyvalues, updatevalues): if keyvalues: - where = "WHERE %s" % " AND ".join("%s = ?" % k for k in keyvalues.iterkeys()) + where = "WHERE %s" % " AND ".join("%s = ?" % k for k in iterkeys(keyvalues)) else: where = "" @@ -802,7 +807,7 @@ class SQLBaseStore(object): txn.execute( update_sql, - updatevalues.values() + keyvalues.values() + list(updatevalues.values()) + list(keyvalues.values()) ) return txn.rowcount @@ -850,7 +855,7 @@ class SQLBaseStore(object): " AND ".join("%s = ?" % (k,) for k in keyvalues) ) - txn.execute(select_sql, keyvalues.values()) + txn.execute(select_sql, list(keyvalues.values())) row = txn.fetchone() if not row: @@ -888,7 +893,7 @@ class SQLBaseStore(object): " AND ".join("%s = ?" % (k, ) for k in keyvalues) ) - txn.execute(sql, keyvalues.values()) + txn.execute(sql, list(keyvalues.values())) if txn.rowcount == 0: raise StoreError(404, "No row found") if txn.rowcount > 1: @@ -906,7 +911,7 @@ class SQLBaseStore(object): " AND ".join("%s = ?" % (k, ) for k in keyvalues) ) - return txn.execute(sql, keyvalues.values()) + return txn.execute(sql, list(keyvalues.values())) def _simple_delete_many(self, table, column, iterable, keyvalues, desc): return self.runInteraction( @@ -938,7 +943,7 @@ class SQLBaseStore(object): ) values.extend(iterable) - for key, value in keyvalues.iteritems(): + for key, value in iteritems(keyvalues): clauses.append("%s = ?" % (key,)) values.append(value) @@ -978,7 +983,7 @@ class SQLBaseStore(object): txn.close() if cache: - min_val = min(cache.itervalues()) + min_val = min(itervalues(cache)) else: min_val = max_value @@ -1093,7 +1098,7 @@ class SQLBaseStore(object): " AND ".join("%s = ?" % (k,) for k in keyvalues), " ? ASC LIMIT ? OFFSET ?" ) - txn.execute(sql, keyvalues.values() + pagevalues) + txn.execute(sql, list(keyvalues.values()) + list(pagevalues)) else: sql = "SELECT %s FROM %s ORDER BY %s" % ( ", ".join(retcols), From 6c16a4ec1b6bc1aa3ef3bbcc080419e1b4bce5bd Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Tue, 15 May 2018 17:45:01 +0200 Subject: [PATCH 254/310] more iteritems --- synapse/storage/roommember.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 6a861943a2..7bfc3d91b5 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -30,6 +30,8 @@ from synapse.types import get_domain_from_id import logging import simplejson as json +from six import itervalues, iteritems + logger = logging.getLogger(__name__) @@ -272,7 +274,7 @@ class RoomMemberWorkerStore(EventsWorkerStore): users_in_room = {} member_event_ids = [ e_id - for key, e_id in current_state_ids.iteritems() + for key, e_id in iteritems(current_state_ids) if key[0] == EventTypes.Member ] @@ -289,7 +291,7 @@ class RoomMemberWorkerStore(EventsWorkerStore): users_in_room = dict(prev_res) member_event_ids = [ e_id - for key, e_id in context.delta_ids.iteritems() + for key, e_id in iteritems(context.delta_ids) if key[0] == EventTypes.Member ] for etype, state_key in context.delta_ids: @@ -741,7 +743,7 @@ class _JoinedHostsCache(object): if state_entry.state_group == self.state_group: pass elif state_entry.prev_group == self.state_group: - for (typ, state_key), event_id in state_entry.delta_ids.iteritems(): + for (typ, state_key), event_id in iteritems(state_entry.delta_ids): if typ != EventTypes.Member: continue @@ -771,7 +773,7 @@ class _JoinedHostsCache(object): self.state_group = state_entry.state_group else: self.state_group = object() - self._len = sum(len(v) for v in self.hosts_to_joined_users.itervalues()) + self._len = sum(len(v) for v in itervalues(self.hosts_to_joined_users)) defer.returnValue(frozenset(self.hosts_to_joined_users)) def __len__(self): From 17a70cf6e9ae016011972b4d8a97aca0c3d945d4 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Sat, 5 May 2018 22:47:18 +0200 Subject: [PATCH 255/310] Misc. py3 fixes Signed-off-by: Adrian Tschira --- synapse/handlers/federation.py | 9 +++++---- synapse/handlers/message.py | 9 +++++---- synapse/storage/events_worker.py | 2 +- synapse/storage/filtering.py | 2 +- synapse/storage/keys.py | 2 +- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 2c72beff2e..87c0615820 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -24,6 +24,7 @@ from signedjson.key import decode_verify_key_bytes from signedjson.sign import verify_signed_json import six from six.moves import http_client +from six import iteritems from twisted.internet import defer from unpaddedbase64 import decode_base64 @@ -1388,7 +1389,7 @@ class FederationHandler(BaseHandler): ) if state_groups: - _, state = state_groups.items().pop() + _, state = list(iteritems(state_groups)).pop() results = { (e.type, e.state_key): e for e in state } @@ -2034,7 +2035,7 @@ class FederationHandler(BaseHandler): this will not be included in the current_state in the context. """ state_updates = { - k: a.event_id for k, a in auth_events.iteritems() + k: a.event_id for k, a in iteritems(auth_events) if k != event_key } context.current_state_ids = dict(context.current_state_ids) @@ -2044,7 +2045,7 @@ class FederationHandler(BaseHandler): context.delta_ids.update(state_updates) context.prev_state_ids = dict(context.prev_state_ids) context.prev_state_ids.update({ - k: a.event_id for k, a in auth_events.iteritems() + k: a.event_id for k, a in iteritems(auth_events) }) context.state_group = yield self.store.store_state_group( event.event_id, @@ -2096,7 +2097,7 @@ class FederationHandler(BaseHandler): def get_next(it, opt=None): try: - return it.next() + return next(it) except Exception: return opt diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index c32b9bcae4..81cff0870e 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -19,6 +19,7 @@ import sys from canonicaljson import encode_canonical_json import six +from six import string_types, itervalues, iteritems from twisted.internet import defer, reactor from twisted.internet.defer import succeed from twisted.python.failure import Failure @@ -402,7 +403,7 @@ class MessageHandler(BaseHandler): "avatar_url": profile.avatar_url, "display_name": profile.display_name, } - for user_id, profile in users_with_profile.iteritems() + for user_id, profile in iteritems(users_with_profile) }) @@ -667,7 +668,7 @@ class EventCreationHandler(object): spam_error = self.spam_checker.check_event_for_spam(event) if spam_error: - if not isinstance(spam_error, basestring): + if not isinstance(spam_error, string_types): spam_error = "Spam is not permitted here" raise SynapseError( 403, spam_error, Codes.FORBIDDEN @@ -881,7 +882,7 @@ class EventCreationHandler(object): state_to_include_ids = [ e_id - for k, e_id in context.current_state_ids.iteritems() + for k, e_id in iteritems(context.current_state_ids) if k[0] in self.hs.config.room_invite_state_types or k == (EventTypes.Member, event.sender) ] @@ -895,7 +896,7 @@ class EventCreationHandler(object): "content": e.content, "sender": e.sender, } - for e in state_to_include.itervalues() + for e in itervalues(state_to_include) ] invitee = UserID.from_string(event.state_key) diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py index ba834854e1..32d9d00ffb 100644 --- a/synapse/storage/events_worker.py +++ b/synapse/storage/events_worker.py @@ -337,7 +337,7 @@ class EventsWorkerStore(SQLBaseStore): def _fetch_event_rows(self, txn, events): rows = [] N = 200 - for i in range(1 + len(events) / N): + for i in range(1 + len(events) // N): evs = events[i * N:(i + 1) * N] if not evs: break diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index 78b1e30945..2e2763126d 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -44,7 +44,7 @@ class FilteringStore(SQLBaseStore): desc="get_user_filter", ) - defer.returnValue(json.loads(str(def_json).decode("utf-8"))) + defer.returnValue(json.loads(bytes(def_json).decode("utf-8"))) def add_user_filter(self, user_localpart, user_filter): def_json = encode_canonical_json(user_filter) diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index 87aeaf71d6..0540c2b0b1 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -92,7 +92,7 @@ class KeyStore(SQLBaseStore): if verify_key_bytes: defer.returnValue(decode_verify_key_bytes( - key_id, str(verify_key_bytes) + key_id, bytes(verify_key_bytes) )) @defer.inlineCallbacks From dd068ca9792e7f4a51690b91262131b5dae80455 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Thu, 24 May 2018 20:52:56 +0200 Subject: [PATCH 256/310] remaining isintance fixes Signed-off-by: Adrian Tschira --- synapse/storage/search.py | 5 +++-- synapse/util/caches/descriptors.py | 12 ++++++++---- synapse/util/frozenutils.py | 5 +++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/synapse/storage/search.py b/synapse/storage/search.py index 6ba3e59889..a9c299a861 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -18,13 +18,14 @@ import logging import re import simplejson as json +from six import string_types + from twisted.internet import defer from .background_updates import BackgroundUpdateStore from synapse.api.errors import SynapseError from synapse.storage.engines import PostgresEngine, Sqlite3Engine - logger = logging.getLogger(__name__) SearchEntry = namedtuple('SearchEntry', [ @@ -126,7 +127,7 @@ class SearchStore(BackgroundUpdateStore): # skip over it. continue - if not isinstance(value, basestring): + if not isinstance(value, string_types): # If the event body, name or topic isn't a string # then skip over it continue diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py index 68285a7594..b595cd6164 100644 --- a/synapse/util/caches/descriptors.py +++ b/synapse/util/caches/descriptors.py @@ -31,6 +31,9 @@ import functools import inspect import threading +from six import string_types, itervalues +import six + logger = logging.getLogger(__name__) @@ -205,7 +208,7 @@ class Cache(object): def invalidate_all(self): self.check_thread() self.cache.clear() - for entry in self._pending_deferred_cache.itervalues(): + for entry in itervalues(self._pending_deferred_cache): entry.invalidate() self._pending_deferred_cache.clear() @@ -392,9 +395,10 @@ class CacheDescriptor(_CacheDescriptorBase): ret.addErrback(onErr) - # If our cache_key is a string, try to convert to ascii to save - # a bit of space in large caches - if isinstance(cache_key, basestring): + # If our cache_key is a string on py2, try to convert to ascii + # to save a bit of space in large caches. Py3 does this + # internally automatically. + if six.PY2 and isinstance(cache_key, string_types): cache_key = to_ascii(cache_key) result_d = ObservableDeferred(ret, consumeErrors=True) diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py index f497b51f4a..4cd0566f4f 100644 --- a/synapse/util/frozenutils.py +++ b/synapse/util/frozenutils.py @@ -16,6 +16,7 @@ from frozendict import frozendict import simplejson as json +from six import string_types def freeze(o): t = type(o) @@ -25,7 +26,7 @@ def freeze(o): if t is frozendict: return o - if t is str or t is unicode: + if isinstance(t, string_types): return o try: @@ -41,7 +42,7 @@ def unfreeze(o): if t is dict or t is frozendict: return dict({k: unfreeze(v) for k, v in o.items()}) - if t is str or t is unicode: + if isinstance(t, string_types): return o try: From 0e617056615d5db9f95ba458b2fafbbe0103c9d8 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Thu, 24 May 2018 20:58:21 +0200 Subject: [PATCH 257/310] py3-ize state.py --- synapse/state.py | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 26093c8434..b8c27c6815 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -32,6 +32,8 @@ from frozendict import frozendict import logging import hashlib +from six import iteritems, itervalues + logger = logging.getLogger(__name__) @@ -132,7 +134,7 @@ class StateHandler(object): state_map = yield self.store.get_events(state.values(), get_prev_content=False) state = { - key: state_map[e_id] for key, e_id in state.iteritems() if e_id in state_map + key: state_map[e_id] for key, e_id in iteritems(state) if e_id in state_map } defer.returnValue(state) @@ -338,7 +340,7 @@ class StateHandler(object): ) if len(state_groups_ids) == 1: - name, state_list = state_groups_ids.items().pop() + name, state_list = list(state_groups_ids.items()).pop() prev_group, delta_ids = yield self.store.get_state_group_delta(name) @@ -378,7 +380,7 @@ class StateHandler(object): new_state = resolve_events_with_state_map(state_set_ids, state_map) new_state = { - key: state_map[ev_id] for key, ev_id in new_state.iteritems() + key: state_map[ev_id] for key, ev_id in iteritems(new_state) } return new_state @@ -458,15 +460,15 @@ class StateResolutionHandler(object): # build a map from state key to the event_ids which set that state. # dict[(str, str), set[str]) state = {} - for st in state_groups_ids.itervalues(): - for key, e_id in st.iteritems(): + for st in itervalues(state_groups_ids): + for key, e_id in iteritems(st): state.setdefault(key, set()).add(e_id) # build a map from state key to the event_ids which set that state, # including only those where there are state keys in conflict. conflicted_state = { k: list(v) - for k, v in state.iteritems() + for k, v in iteritems(state) if len(v) > 1 } @@ -474,13 +476,13 @@ class StateResolutionHandler(object): logger.info("Resolving conflicted state for %r", room_id) with Measure(self.clock, "state._resolve_events"): new_state = yield resolve_events_with_factory( - state_groups_ids.values(), + list(state_groups_ids.values()), event_map=event_map, state_map_factory=state_map_factory, ) else: new_state = { - key: e_ids.pop() for key, e_ids in state.iteritems() + key: e_ids.pop() for key, e_ids in iteritems(state) } with Measure(self.clock, "state.create_group_ids"): @@ -489,8 +491,8 @@ class StateResolutionHandler(object): # which will be used as a cache key for future resolutions, but # not get persisted. state_group = None - new_state_event_ids = frozenset(new_state.itervalues()) - for sg, events in state_groups_ids.iteritems(): + new_state_event_ids = frozenset(itervalues(new_state)) + for sg, events in iteritems(state_groups_ids): if new_state_event_ids == frozenset(e_id for e_id in events): state_group = sg break @@ -501,11 +503,11 @@ class StateResolutionHandler(object): prev_group = None delta_ids = None - for old_group, old_ids in state_groups_ids.iteritems(): + for old_group, old_ids in iteritems(state_groups_ids): if not set(new_state) - set(old_ids): n_delta_ids = { k: v - for k, v in new_state.iteritems() + for k, v in iteritems(new_state) if old_ids.get(k) != v } if not delta_ids or len(n_delta_ids) < len(delta_ids): @@ -527,7 +529,7 @@ class StateResolutionHandler(object): def _ordered_events(events): def key_func(e): - return -int(e.depth), hashlib.sha1(e.event_id).hexdigest() + return -int(e.depth), hashlib.sha1(e.event_id.encode()).hexdigest() return sorted(events, key=key_func) @@ -584,7 +586,7 @@ def _seperate(state_sets): conflicted_state = {} for state_set in state_sets[1:]: - for key, value in state_set.iteritems(): + for key, value in iteritems(state_set): # Check if there is an unconflicted entry for the state key. unconflicted_value = unconflicted_state.get(key) if unconflicted_value is None: @@ -640,7 +642,7 @@ def resolve_events_with_factory(state_sets, event_map, state_map_factory): needed_events = set( event_id - for event_ids in conflicted_state.itervalues() + for event_ids in itervalues(conflicted_state) for event_id in event_ids ) if event_map is not None: @@ -662,7 +664,7 @@ def resolve_events_with_factory(state_sets, event_map, state_map_factory): unconflicted_state, conflicted_state, state_map ) - new_needed_events = set(auth_events.itervalues()) + new_needed_events = set(itervalues(auth_events)) new_needed_events -= needed_events if event_map is not None: new_needed_events -= set(event_map.iterkeys()) @@ -679,7 +681,7 @@ def resolve_events_with_factory(state_sets, event_map, state_map_factory): def _create_auth_events_from_maps(unconflicted_state, conflicted_state, state_map): auth_events = {} - for event_ids in conflicted_state.itervalues(): + for event_ids in itervalues(conflicted_state): for event_id in event_ids: if event_id in state_map: keys = event_auth.auth_types_for_event(state_map[event_id]) @@ -694,7 +696,7 @@ def _create_auth_events_from_maps(unconflicted_state, conflicted_state, state_ma def _resolve_with_state(unconflicted_state_ids, conflicted_state_ds, auth_event_ids, state_map): conflicted_state = {} - for key, event_ids in conflicted_state_ds.iteritems(): + for key, event_ids in iteritems(conflicted_state_ds): events = [state_map[ev_id] for ev_id in event_ids if ev_id in state_map] if len(events) > 1: conflicted_state[key] = events @@ -703,7 +705,7 @@ def _resolve_with_state(unconflicted_state_ids, conflicted_state_ds, auth_event_ auth_events = { key: state_map[ev_id] - for key, ev_id in auth_event_ids.iteritems() + for key, ev_id in iteritems(auth_event_ids) if ev_id in state_map } @@ -716,7 +718,7 @@ def _resolve_with_state(unconflicted_state_ids, conflicted_state_ds, auth_event_ raise new_state = unconflicted_state_ids - for key, event in resolved_state.iteritems(): + for key, event in iteritems(resolved_state): new_state[key] = event.event_id return new_state @@ -741,7 +743,7 @@ def _resolve_state_events(conflicted_state, auth_events): auth_events.update(resolved_state) - for key, events in conflicted_state.iteritems(): + for key, events in iteritems(conflicted_state): if key[0] == EventTypes.JoinRules: logger.debug("Resolving conflicted join rules %r", events) resolved_state[key] = _resolve_auth_events( @@ -751,7 +753,7 @@ def _resolve_state_events(conflicted_state, auth_events): auth_events.update(resolved_state) - for key, events in conflicted_state.iteritems(): + for key, events in iteritems(conflicted_state): if key[0] == EventTypes.Member: logger.debug("Resolving conflicted member lists %r", events) resolved_state[key] = _resolve_auth_events( @@ -761,7 +763,7 @@ def _resolve_state_events(conflicted_state, auth_events): auth_events.update(resolved_state) - for key, events in conflicted_state.iteritems(): + for key, events in iteritems(conflicted_state): if key not in resolved_state: logger.debug("Resolving conflicted state %r:%r", key, events) resolved_state[key] = _resolve_normal_events( From 4ee4450d66c3c0230a4d81909220a0a94af8be8f Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Thu, 24 May 2018 21:44:10 +0200 Subject: [PATCH 258/310] fix recursion error --- synapse/util/frozenutils.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py index 4cd0566f4f..97914907aa 100644 --- a/synapse/util/frozenutils.py +++ b/synapse/util/frozenutils.py @@ -19,14 +19,13 @@ import simplejson as json from six import string_types def freeze(o): - t = type(o) - if t is dict: + if isinstance(o, dict): return frozendict({k: freeze(v) for k, v in o.items()}) - if t is frozendict: + if isinstance(o, frozendict): return o - if isinstance(t, string_types): + if isinstance(o, string_types): return o try: @@ -38,11 +37,10 @@ def freeze(o): def unfreeze(o): - t = type(o) - if t is dict or t is frozendict: + if isinstance(o, (dict, frozendict)): return dict({k: unfreeze(v) for k, v in o.items()}) - if isinstance(t, string_types): + if isinstance(o, string_types): return o try: From 757ed2725803ff6d190eb1bb8dea872345862aa3 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 25 May 2018 11:07:21 +0100 Subject: [PATCH 259/310] Let users leave the server notice room after joining They still can't reject invites, but we let them leave it. --- docs/server_notices.md | 7 +++++-- synapse/handlers/room_member.py | 24 ++++++++++++++---------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/docs/server_notices.md b/docs/server_notices.md index 221553b24d..58f8776319 100644 --- a/docs/server_notices.md +++ b/docs/server_notices.md @@ -5,7 +5,7 @@ Server Notices channel whereby server administrators can send messages to users on the server. They are used as part of communication of the server polices(see -[consent_tracking.md](consent_tracking.md)), however the intention is that +[consent_tracking.md](consent_tracking.md)), however the intention is that they may also find a use for features such as "Message of the day". This is a feature specific to Synapse, but it uses standard Matrix @@ -24,7 +24,10 @@ history; it will appear to have come from the 'server notices user' (see below). The user is prevented from sending any messages in this room by the power -levels. They also cannot leave it. +levels. + +Having joined the room, the user can leave the room if they want. Subsequent +server notices will then cause a new room to be created. Synapse configuration --------------------- diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 82adfc8fdf..f930e939e8 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -298,16 +298,6 @@ class RoomMemberHandler(object): is_blocked = yield self.store.is_room_blocked(room_id) if is_blocked: raise SynapseError(403, "This room has been blocked on this server") - else: - # we don't allow people to reject invites to, or leave, the - # server notice room. - is_blocked = yield self._is_server_notice_room(room_id) - if is_blocked: - raise SynapseError( - http_client.FORBIDDEN, - "You cannot leave this room", - errcode=Codes.CANNOT_LEAVE_SERVER_NOTICE_ROOM, - ) if effective_membership_state == Membership.INVITE: # block any attempts to invite the server notices mxid @@ -383,6 +373,20 @@ class RoomMemberHandler(object): if same_sender and same_membership and same_content: defer.returnValue(old_state) + # we don't allow people to reject invites to the server notice + # room, but they can leave it once they are joined. + if ( + old_membership == Membership.INVITE and + effective_membership_state == Membership.LEAVE + ): + is_blocked = yield self._is_server_notice_room(room_id) + if is_blocked: + raise SynapseError( + http_client.FORBIDDEN, + "You cannot reject this invite", + errcode=Codes.CANNOT_LEAVE_SERVER_NOTICE_ROOM, + ) + is_host_in_room = yield self._is_host_in_room(current_state_ids) if effective_membership_state == Membership.JOIN: From ba1b1635901f148d793777ea77386dbe3412dfaf Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 25 May 2018 11:36:43 +0100 Subject: [PATCH 260/310] Avoid sending consent notice to guest users we think it makes sense not to send the notices to guest users. --- synapse/config/consent_config.py | 8 +++++++- synapse/server_notices/consent_server_notices.py | 5 +++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py index ddcd305f4c..3f29346593 100644 --- a/synapse/config/consent_config.py +++ b/synapse/config/consent_config.py @@ -32,7 +32,8 @@ DEFAULT_CONFIG = """\ # # 'server_notice_content', if enabled, will send a user a "Server Notice" # asking them to consent to the privacy policy. The 'server_notices' section -# must also be configured for this to work. +# must also be configured for this to work. Notices will *not* be sent to +# guest users unless 'send_server_notice_to_guests' is set to true. # # 'block_events_error', if set, will block any attempts to send events # until the user consents to the privacy policy. The value of the setting is @@ -46,6 +47,7 @@ DEFAULT_CONFIG = """\ # body: >- # To continue using this homeserver you must review and agree to the # terms and conditions at %(consent_uri)s +# send_server_notice_to_guests: True # block_events_error: >- # To continue using this homeserver you must review and agree to the # terms and conditions at %(consent_uri)s @@ -60,6 +62,7 @@ class ConsentConfig(Config): self.user_consent_version = None self.user_consent_template_dir = None self.user_consent_server_notice_content = None + self.user_consent_server_notice_to_guests = False self.block_events_without_consent_error = None def read_config(self, config): @@ -74,6 +77,9 @@ class ConsentConfig(Config): self.block_events_without_consent_error = consent_config.get( "block_events_error", ) + self.user_consent_server_notice_to_guests = bool(consent_config.get( + "send_server_notice_to_guests", "False" + )) def default_config(self, **kwargs): return DEFAULT_CONFIG diff --git a/synapse/server_notices/consent_server_notices.py b/synapse/server_notices/consent_server_notices.py index a709802856..bb74af1af5 100644 --- a/synapse/server_notices/consent_server_notices.py +++ b/synapse/server_notices/consent_server_notices.py @@ -42,6 +42,7 @@ class ConsentServerNotices(object): self._current_consent_version = hs.config.user_consent_version self._server_notice_content = hs.config.user_consent_server_notice_content + self._send_to_guests = hs.config.user_consent_server_notice_to_guests if self._server_notice_content is not None: if not self._server_notices_manager.is_enabled(): @@ -78,6 +79,10 @@ class ConsentServerNotices(object): try: u = yield self._store.get_user_by_id(user_id) + if u["is_guest"] and not self._send_to_guests: + # don't send to guests + return + if u["consent_version"] == self._current_consent_version: # user has already consented return From 66bdae986f4bdee300aebb4ad6ebeeba48286e6b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 25 May 2018 11:42:05 +0100 Subject: [PATCH 261/310] Fix default for send_server_notice_to_guests bool("False") == True... --- synapse/config/consent_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py index 3f29346593..8f6ed73328 100644 --- a/synapse/config/consent_config.py +++ b/synapse/config/consent_config.py @@ -78,7 +78,7 @@ class ConsentConfig(Config): "block_events_error", ) self.user_consent_server_notice_to_guests = bool(consent_config.get( - "send_server_notice_to_guests", "False" + "send_server_notice_to_guests", False, )) def default_config(self, **kwargs): From a2eb5db4a066148da5057b8ee844c7e4f5d888bc Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 28 May 2018 19:10:27 +1000 Subject: [PATCH 262/310] update metrics to be in seconds --- synapse/http/request_metrics.py | 6 +++--- synapse/http/site.py | 20 ++++++++++---------- synapse/metrics/__init__.py | 19 ++++++++++--------- synapse/storage/_base.py | 30 +++++++++++++++--------------- 4 files changed, 38 insertions(+), 37 deletions(-) diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py index e7df494333..af3067b4bb 100644 --- a/synapse/http/request_metrics.py +++ b/synapse/http/request_metrics.py @@ -38,15 +38,15 @@ outgoing_responses_counter = Counter( ) response_timer = Histogram( - "synapse_http_server_response_time_seconds", "", ["method", "servlet", "tag"] + "synapse_http_server_response_time_seconds", "sec", ["method", "servlet", "tag"] ) response_ru_utime = Counter( - "synapse_http_server_response_ru_utime_seconds", "", ["method", "servlet", "tag"] + "synapse_http_server_response_ru_utime_seconds", "sec", ["method", "servlet", "tag"] ) response_ru_stime = Counter( - "synapse_http_server_response_ru_stime_seconds", "", ["method", "servlet", "tag"] + "synapse_http_server_response_ru_stime_seconds", "sec", ["method", "servlet", "tag"] ) response_db_txn_count = Counter( diff --git a/synapse/http/site.py b/synapse/http/site.py index 23c1b76922..60299657b9 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -83,7 +83,7 @@ class SynapseRequest(Request): return Request.render(self, resrc) def _started_processing(self, servlet_name): - self.start_time = int(time.time() * 1000) + self.start_time = time.time() self.request_metrics = RequestMetrics() self.request_metrics.start( self.start_time, name=servlet_name, method=self.method, @@ -102,26 +102,26 @@ class SynapseRequest(Request): context = LoggingContext.current_context() ru_utime, ru_stime = context.get_resource_usage() db_txn_count = context.db_txn_count - db_txn_duration_ms = context.db_txn_duration_ms - db_sched_duration_ms = context.db_sched_duration_ms + db_txn_duration_sec = context.db_txn_duration_sec + db_sched_duration_sec = context.db_sched_duration_sec except Exception: ru_utime, ru_stime = (0, 0) - db_txn_count, db_txn_duration_ms = (0, 0) + db_txn_count, db_txn_duration_sec = (0, 0) - end_time = int(time.time() * 1000) + end_time = time.time() self.site.access_logger.info( "%s - %s - {%s}" - " Processed request: %dms (%dms, %dms) (%dms/%dms/%d)" + " Processed request: %.3fsec (%.3fsec, %.3fsec) (%.3fsec/%.3fsec/%d)" " %sB %s \"%s %s %s\" \"%s\"", self.getClientIP(), self.site.site_tag, self.authenticated_entity, end_time - self.start_time, - int(ru_utime * 1000), - int(ru_stime * 1000), - db_sched_duration_ms, - db_txn_duration_ms, + ru_utime, + ru_stime, + db_sched_duration_sec, + db_txn_duration_sec, int(db_txn_count), self.sentLength, self.code, diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index e33ed6c9be..0c557a43f6 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -122,10 +122,10 @@ REGISTRY.register(CPUMetrics()) gc_unreachable = Gauge("python_gc_unreachable_total", "Unreachable GC objects", ["gen"]) gc_time = Histogram( "python_gc_time", - "Time taken to GC (ms)", + "Time taken to GC (sec)", ["gen"], - buckets=[2.5, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 7500, 15000, - 30000, 45000, 60000], + buckets=[0.0025, 0.005, 0.01, 0.025, 0.05, 0.10, 0.25, 0.50, 1.00, 2.50, + 5.00, 7.50, 15.00, 30.00, 45.00, 60.00], ) @@ -147,8 +147,9 @@ REGISTRY.register(GCCounts()) tick_time = Histogram( "python_twisted_reactor_tick_time", - "Tick time of the Twisted reactor (ms)", - buckets=[1, 2, 5, 10, 50, 100, 250, 500, 1000, 2000], + "Tick time of the Twisted reactor (sec)", + buckets=[0.001, 0.002, 0.005, 0.001, 0.005, 0.01. 0.025, 0.05, 0.1, 0.2, + 0.5, 1, 2, 5], ) pending_calls_metric = Histogram( "python_twisted_reactor_pending_calls", @@ -202,9 +203,9 @@ def runUntilCurrentTimer(func): num_pending += 1 num_pending += len(reactor.threadCallQueue) - start = time.time() * 1000 + start = time.time() ret = func(*args, **kwargs) - end = time.time() * 1000 + end = time.time() # record the amount of wallclock time spent running pending calls. # This is a proxy for the actual amount of time between reactor polls, @@ -225,9 +226,9 @@ def runUntilCurrentTimer(func): if threshold[i] < counts[i]: logger.info("Collecting gc %d", i) - start = time.time() * 1000 + start = time.time() unreachable = gc.collect(i) - end = time.time() * 1000 + end = time.time() gc_time.labels(i).observe(end - start) gc_unreachable.labels(i).set(unreachable) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index d963af5c89..22d6257a9f 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -42,10 +42,10 @@ sql_logger = logging.getLogger("synapse.storage.SQL") transaction_logger = logging.getLogger("synapse.storage.txn") perf_logger = logging.getLogger("synapse.storage.TIME") -sql_scheduling_timer = Histogram("synapse_storage_schedule_time", "") +sql_scheduling_timer = Histogram("synapse_storage_schedule_time", "sec") -sql_query_timer = Histogram("synapse_storage_query_time", "", ["verb"]) -sql_txn_timer = Histogram("synapse_storage_transaction_time", "", ["desc"]) +sql_query_timer = Histogram("synapse_storage_query_time", "sec", ["verb"]) +sql_txn_timer = Histogram("synapse_storage_transaction_time", "sec", ["desc"]) class LoggingTransaction(object): @@ -110,7 +110,7 @@ class LoggingTransaction(object): # Don't let logging failures stop SQL from working pass - start = time.time() * 1000 + start = time.time() try: return func( @@ -120,9 +120,9 @@ class LoggingTransaction(object): logger.debug("[SQL FAIL] {%s} %s", self.name, e) raise finally: - msecs = (time.time() * 1000) - start - sql_logger.debug("[SQL time] {%s} %f", self.name, msecs) - sql_query_timer.labels(sql.split()[0]).observe(msecs) + secs = time.time() - start + sql_logger.debug("[SQL time] {%s} %f sec", self.name, secs) + sql_query_timer.labels(sql.split()[0]).observe(secs) class PerformanceCounters(object): @@ -132,7 +132,7 @@ class PerformanceCounters(object): def update(self, key, start_time, end_time=None): if end_time is None: - end_time = time.time() * 1000 + end_time = time.time() duration = end_time - start_time count, cum_time = self.current_counters.get(key, (0, 0)) count += 1 @@ -222,7 +222,7 @@ class SQLBaseStore(object): def _new_transaction(self, conn, desc, after_callbacks, exception_callbacks, logging_context, func, *args, **kwargs): - start = time.time() * 1000 + start = time.time() txn_id = self._TXN_ID # We don't really need these to be unique, so lets stop it from @@ -282,13 +282,13 @@ class SQLBaseStore(object): logger.debug("[TXN FAIL] {%s} %s", name, e) raise finally: - end = time.time() * 1000 + end = time.time() duration = end - start if logging_context is not None: logging_context.add_database_transaction(duration) - transaction_logger.debug("[TXN END] {%s} %f", name, duration) + transaction_logger.debug("[TXN END] {%s} %f sec", name, duration) self._current_txn_total_time += duration self._txn_perf_counters.update(desc, start, end) @@ -349,13 +349,13 @@ class SQLBaseStore(object): """ current_context = LoggingContext.current_context() - start_time = time.time() * 1000 + start_time = time.time() def inner_func(conn, *args, **kwargs): with LoggingContext("runWithConnection") as context: - sched_duration_ms = time.time() * 1000 - start_time - sql_scheduling_timer.observe(sched_duration_ms) - current_context.add_database_scheduled(sched_duration_ms) + sched_duration_sec = time.time() - start_time + sql_scheduling_timer.observe(sched_duration_sec) + current_context.add_database_scheduled(sched_duration_sec) if self.database_engine.is_connection_closed(conn): logger.debug("Reconnecting closed database connection") From 357c74a50f1e1588a6a3d626bddb3555452c6f56 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 28 May 2018 19:14:41 +1000 Subject: [PATCH 263/310] add comment about why unreg --- synapse/util/caches/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index d968b71e7a..183faf75a1 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -41,6 +41,8 @@ response_cache_total = Gauge("synapse_util_caches_response_cache:total", "", ["n def register_cache(cache_type, cache_name, cache): # Check if the metric is already registered. Unregister it, if so. + # This usually happens during tests, as at runtime these caches are + # effectively singletons. metric_name = "cache_%s_%s" % (cache_type, cache_name) if metric_name in collectors_by_name.keys(): REGISTRY.unregister(collectors_by_name[metric_name]) From 5c40ce37774a20b57fe726b9c3d880d089ed6f0c Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 28 May 2018 19:16:09 +1000 Subject: [PATCH 264/310] invalid syntax :( --- synapse/metrics/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index 0c557a43f6..371c300cac 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -148,8 +148,7 @@ REGISTRY.register(GCCounts()) tick_time = Histogram( "python_twisted_reactor_tick_time", "Tick time of the Twisted reactor (sec)", - buckets=[0.001, 0.002, 0.005, 0.001, 0.005, 0.01. 0.025, 0.05, 0.1, 0.2, - 0.5, 1, 2, 5], + buckets=[0.001, 0.002, 0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2, 5], ) pending_calls_metric = Histogram( "python_twisted_reactor_pending_calls", From 3ef5cd74a6bff9b33144cb834782e0402e6eb152 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 28 May 2018 19:39:27 +1000 Subject: [PATCH 265/310] update to more consistently use seconds in any metrics or logging --- synapse/http/request_metrics.py | 38 ++++++++++++++++----------------- synapse/util/logcontext.py | 24 ++++++++++----------- synapse/util/logutils.py | 6 +++--- synapse/util/metrics.py | 8 +++---- 4 files changed, 38 insertions(+), 38 deletions(-) diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py index af3067b4bb..dc06f6c443 100644 --- a/synapse/http/request_metrics.py +++ b/synapse/http/request_metrics.py @@ -139,8 +139,8 @@ LaterGauge( class RequestMetrics(object): - def start(self, time_msec, name, method): - self.start = time_msec + def start(self, time_sec, name, method): + self.start = time_sec self.start_context = LoggingContext.current_context() self.name = name self.method = method @@ -149,7 +149,7 @@ class RequestMetrics(object): _in_flight_requests.add(self) - def stop(self, time_msec, request): + def stop(self, time_sec, request): _in_flight_requests.discard(self) context = LoggingContext.current_context() @@ -170,7 +170,7 @@ class RequestMetrics(object): response_count.labels(request.method, self.name, tag).inc() response_timer.labels(request.method, self.name, tag).observe( - time_msec - self.start + time_sec - self.start ) ru_utime, ru_stime = context.get_resource_usage() @@ -181,10 +181,10 @@ class RequestMetrics(object): context.db_txn_count ) response_db_txn_duration.labels(request.method, self.name, tag).inc( - context.db_txn_duration_ms / 1000. + context.db_txn_duration_sec ) response_db_sched_duration.labels(request.method, self.name, tag).inc( - context.db_sched_duration_ms / 1000. + context.db_sched_duration_sec ) response_size.labels(request.method, self.name, tag).inc(request.sentLength) @@ -207,11 +207,11 @@ class RequestMetrics(object): ) in_flight_requests_db_txn_duration.labels(self.method, self.name).inc( - diff.db_txn_duration_ms / 1000. + diff.db_txn_duration_sec ) in_flight_requests_db_sched_duration.labels(self.method, self.name).inc( - diff.db_sched_duration_ms / 1000. + diff.db_sched_duration_sec ) @@ -223,18 +223,18 @@ class _RequestStats(object): "ru_utime", "ru_stime", "db_txn_count", - "db_txn_duration_ms", - "db_sched_duration_ms", + "db_txn_duration_sec", + "db_sched_duration_sec", ] def __init__( - self, ru_utime, ru_stime, db_txn_count, db_txn_duration_ms, db_sched_duration_ms + self, ru_utime, ru_stime, db_txn_count, db_txn_duration_sec, db_sched_duration_sec ): self.ru_utime = ru_utime self.ru_stime = ru_stime self.db_txn_count = db_txn_count - self.db_txn_duration_ms = db_txn_duration_ms - self.db_sched_duration_ms = db_sched_duration_ms + self.db_txn_duration_sec = db_txn_duration_sec + self.db_sched_duration_sec = db_sched_duration_sec @staticmethod def from_context(context): @@ -243,8 +243,8 @@ class _RequestStats(object): return _RequestStats( ru_utime, ru_stime, context.db_txn_count, - context.db_txn_duration_ms, - context.db_sched_duration_ms, + context.db_txn_duration_sec, + context.db_sched_duration_sec, ) def update(self, context): @@ -260,14 +260,14 @@ class _RequestStats(object): new.ru_utime - self.ru_utime, new.ru_stime - self.ru_stime, new.db_txn_count - self.db_txn_count, - new.db_txn_duration_ms - self.db_txn_duration_ms, - new.db_sched_duration_ms - self.db_sched_duration_ms, + new.db_txn_duration_sec - self.db_txn_duration_sec, + new.db_sched_duration_sec - self.db_sched_duration_sec, ) self.ru_utime = new.ru_utime self.ru_stime = new.ru_stime self.db_txn_count = new.db_txn_count - self.db_txn_duration_ms = new.db_txn_duration_ms - self.db_sched_duration_ms = new.db_sched_duration_ms + self.db_txn_duration_sec = new.db_txn_duration_sec + self.db_sched_duration_sec = new.db_sched_duration_sec return diff diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index 914f616312..a58c723403 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -59,7 +59,7 @@ class LoggingContext(object): __slots__ = [ "previous_context", "name", "ru_stime", "ru_utime", - "db_txn_count", "db_txn_duration_ms", "db_sched_duration_ms", + "db_txn_count", "db_txn_duration_sec", "db_sched_duration_sec", "usage_start", "main_thread", "alive", "request", "tag", @@ -84,10 +84,10 @@ class LoggingContext(object): def stop(self): pass - def add_database_transaction(self, duration_ms): + def add_database_transaction(self, duration_sec): pass - def add_database_scheduled(self, sched_ms): + def add_database_scheduled(self, sched_sec): pass def __nonzero__(self): @@ -103,11 +103,11 @@ class LoggingContext(object): self.ru_utime = 0. self.db_txn_count = 0 - # ms spent waiting for db txns, excluding scheduling time - self.db_txn_duration_ms = 0 + # sec spent waiting for db txns, excluding scheduling time + self.db_txn_duration_sec = 0 - # ms spent waiting for db txns to be scheduled - self.db_sched_duration_ms = 0 + # sec spent waiting for db txns to be scheduled + self.db_sched_duration_sec = 0 # If alive has the thread resource usage when the logcontext last # became active. @@ -230,18 +230,18 @@ class LoggingContext(object): return ru_utime, ru_stime - def add_database_transaction(self, duration_ms): + def add_database_transaction(self, duration_sec): self.db_txn_count += 1 - self.db_txn_duration_ms += duration_ms + self.db_txn_duration_sec += duration_sec - def add_database_scheduled(self, sched_ms): + def add_database_scheduled(self, sched_sec): """Record a use of the database pool Args: - sched_ms (int): number of milliseconds it took us to get a + sched_sec (float): number of seconds it took us to get a connection """ - self.db_sched_duration_ms += sched_ms + self.db_sched_duration_sec += sched_sec class LoggingContextFilter(logging.Filter): diff --git a/synapse/util/logutils.py b/synapse/util/logutils.py index 3a83828d25..03249c5dc8 100644 --- a/synapse/util/logutils.py +++ b/synapse/util/logutils.py @@ -96,7 +96,7 @@ def time_function(f): id = _TIME_FUNC_ID _TIME_FUNC_ID += 1 - start = time.clock() * 1000 + start = time.clock() try: _log_debug_as_f( @@ -107,10 +107,10 @@ def time_function(f): r = f(*args, **kwargs) finally: - end = time.clock() * 1000 + end = time.clock() _log_debug_as_f( f, - "[FUNC END] {%s-%d} %f", + "[FUNC END] {%s-%d} %.3f sec", (func_name, id, end - start,), ) diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index 424fdcb036..23fc0ca20d 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -74,7 +74,7 @@ class Measure(object): self.created_context = False def __enter__(self): - self.start = self.clock.time_msec() + self.start = self.clock.time() self.start_context = LoggingContext.current_context() if not self.start_context: self.start_context = LoggingContext("Measure") @@ -90,7 +90,7 @@ class Measure(object): if isinstance(exc_type, Exception) or not self.start_context: return - duration = self.clock.time_msec() - self.start + duration = self.clock.time() - self.start block_counter.labels(self.name).inc() block_timer.labels(self.name).inc(duration) @@ -114,9 +114,9 @@ class Measure(object): block_ru_stime.labels(self.name).inc(ru_stime - self.ru_stime) block_db_txn_count.labels(self.name).inc(context.db_txn_count - self.db_txn_count) block_db_txn_duration.labels(self.name).inc( - (context.db_txn_duration_ms - self.db_txn_duration_ms) / 1000.) + context.db_txn_duration_sec - self.db_txn_duration_sec) block_db_sched_duration.labels(self.name).inc( - (context.db_sched_duration_ms - self.db_sched_duration_ms) / 1000.) + context.db_sched_duration_sec - self.db_sched_duration_sec) if self.created_context: self.start_context.__exit__(exc_type, exc_val, exc_tb) From 57ad76fa4a4fb76dc1b9c7e8232b9589b9731ff6 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 28 May 2018 19:51:53 +1000 Subject: [PATCH 266/310] fix up tests --- synapse/util/metrics.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index 23fc0ca20d..1ba7d65c7c 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -62,7 +62,7 @@ class Measure(object): __slots__ = [ "clock", "name", "start_context", "start", "new_context", "ru_utime", "ru_stime", - "db_txn_count", "db_txn_duration_ms", "db_sched_duration_ms", + "db_txn_count", "db_txn_duration_sec", "db_sched_duration_sec", "created_context", ] @@ -83,8 +83,8 @@ class Measure(object): self.ru_utime, self.ru_stime = self.start_context.get_resource_usage() self.db_txn_count = self.start_context.db_txn_count - self.db_txn_duration_ms = self.start_context.db_txn_duration_ms - self.db_sched_duration_ms = self.start_context.db_sched_duration_ms + self.db_txn_duration_sec = self.start_context.db_txn_duration_sec + self.db_sched_duration_sec = self.start_context.db_sched_duration_sec def __exit__(self, exc_type, exc_val, exc_tb): if isinstance(exc_type, Exception) or not self.start_context: From 284e36ad0443eaa341e4b32abb15a5a90185dffe Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 29 May 2018 02:21:52 +0100 Subject: [PATCH 267/310] fix dumb typo --- tests/test_dns.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_dns.py b/tests/test_dns.py index af607d626f..3b360a0fc7 100644 --- a/tests/test_dns.py +++ b/tests/test_dns.py @@ -62,7 +62,7 @@ class DnsTestCase(unittest.TestCase): dns_client_mock = Mock() dns_client_mock.lookupService.return_value = defer.fail(error.DNSServerError()) - service_name = "test_service.examle.com" + service_name = "test_service.example.com" entry = Mock(spec_set=["expires"]) entry.expires = 0 @@ -87,7 +87,7 @@ class DnsTestCase(unittest.TestCase): dns_client_mock = Mock(spec_set=['lookupService']) dns_client_mock.lookupService = Mock(spec_set=[]) - service_name = "test_service.examle.com" + service_name = "test_service.example.com" entry = Mock(spec_set=["expires"]) entry.expires = 999999999 @@ -111,7 +111,7 @@ class DnsTestCase(unittest.TestCase): dns_client_mock.lookupService.return_value = defer.fail(error.DNSServerError()) - service_name = "test_service.examle.com" + service_name = "test_service.example.com" cache = {} @@ -126,7 +126,7 @@ class DnsTestCase(unittest.TestCase): dns_client_mock.lookupService.return_value = defer.fail(error.DNSNameError()) - service_name = "test_service.examle.com" + service_name = "test_service.example.com" cache = {} From 0a240ad36eec189b5a85cc3f2c215e3e5898d5b7 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 29 May 2018 02:22:27 +0100 Subject: [PATCH 268/310] disable CPUMetrics if no /proc/self/stat fixes build on macOS again --- synapse/metrics/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index 371c300cac..6b5b0c9471 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -34,6 +34,7 @@ all_metrics = [] all_collectors = [] all_gauges = {} +HAVE_PROC_SELF_STAT = os.path.exists("/proc/self/stat") class RegistryProxy(object): @@ -99,6 +100,8 @@ class CPUMetrics(object): self.ticks_per_sec = ticks_per_sec def collect(self): + if not HAVE_PROC_SELF_STAT: + return with open("/proc/self/stat") as s: line = s.read() From adb6bac4d5590afcf8a29586ecc69576f98d3537 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 29 May 2018 02:29:22 +0100 Subject: [PATCH 269/310] fix another dumb typo --- tests/util/test_wheel_timer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/util/test_wheel_timer.py b/tests/util/test_wheel_timer.py index c44567e52e..fdb24a48b0 100644 --- a/tests/util/test_wheel_timer.py +++ b/tests/util/test_wheel_timer.py @@ -33,7 +33,7 @@ class WheelTimerTestCase(unittest.TestCase): self.assertListEqual(wheel.fetch(156), [obj]) self.assertListEqual(wheel.fetch(170), []) - def test_mutli_insert(self): + def test_multi_insert(self): wheel = WheelTimer(bucket_size=5) obj1 = object() @@ -58,7 +58,7 @@ class WheelTimerTestCase(unittest.TestCase): wheel.insert(100, obj, 50) self.assertListEqual(wheel.fetch(120), [obj]) - def test_insert_past_mutli(self): + def test_insert_past_multi(self): wheel = WheelTimer(bucket_size=5) obj1 = object() From ff1bc0a2796b07558f252f3bd2645c2b5c5088f4 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 29 May 2018 02:32:15 +0100 Subject: [PATCH 270/310] pep8 --- synapse/metrics/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index 6b5b0c9471..bfdbbc9a23 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -36,6 +36,7 @@ all_gauges = {} HAVE_PROC_SELF_STAT = os.path.exists("/proc/self/stat") + class RegistryProxy(object): def collect(self): From d6cd55532b547892a649a2895983cb3e845ea5d4 Mon Sep 17 00:00:00 2001 From: Krombel Date: Tue, 29 May 2018 16:02:41 +0200 Subject: [PATCH 271/310] extend tox testing for py3 to avoid regressions --- tox.ini | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 3e114e382c..99b35f399a 100644 --- a/tox.ini +++ b/tox.ini @@ -51,7 +51,34 @@ usedevelop=true commands = /usr/bin/find "{toxinidir}" -name '*.pyc' -delete coverage run {env:COVERAGE_OPTS:} --source="{toxinidir}/synapse" \ - "{envbindir}/trial" {env:TRIAL_FLAGS:} {posargs:tests/config} \ + "{envbindir}/trial" {env:TRIAL_FLAGS:} {posargs:tests/config \ + tests/appservice/test_scheduler.py \ + tests/handlers/test_auth.py \ + tests/handlers/test_presence.py \ + tests/handlers/test_register.py \ + tests/storage/test_appservice.py \ + tests/storage/test_base.py \ + tests/storage/test_client_ips.py \ + tests/storage/test_devices.py \ + tests/storage/test_end_to_end_keys.py \ + tests/storage/test_event_push_actions.py \ + tests/storage/test_profile.py \ + tests/storage/test_room.py \ + tests/test_distributor.py \ + tests/test_dns.py \ + tests/test_preview.py \ + tests/test_test_utils.py \ + tests/test_types.py \ + tests/util/test_dict_cache.py \ + tests/util/test_expiring_cache.py \ + tests/util/test_file_consumer.py \ + tests/util/test_limiter.py \ + tests/util/test_linearizer.py \ + tests/util/test_logcontext.py \ + tests/util/test_logformatter.py \ + tests/util/test_rwlock.py \ + tests/util/test_snapshot_cache.py \ + tests/util/test_wheel_timer.py} \ {env:TOXSUFFIX:} {env:DUMP_COVERAGE_COMMAND:coverage report -m} From 1afafb34978726b63c2a02520ff20d85382ce2bc Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Tue, 29 May 2018 17:10:06 +0200 Subject: [PATCH 272/310] use memoryview in py3 Signed-off-by: Adrian Tschira --- synapse/storage/keys.py | 8 ++++++++ synapse/storage/signatures.py | 12 ++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index 0540c2b0b1..e56c464852 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -17,6 +17,7 @@ from ._base import SQLBaseStore from synapse.util.caches.descriptors import cachedInlineCallbacks from twisted.internet import defer +import six import OpenSSL from signedjson.key import decode_verify_key_bytes @@ -26,6 +27,13 @@ import logging logger = logging.getLogger(__name__) +# py2 sqlite has buffer hardcoded as only binary type, so we must use it, +# despite being deprecated and removed in favor of memoryview +if six.PY2: + db_binary_type = buffer +else: + db_binary_type = memoryview + class KeyStore(SQLBaseStore): """Persistence for signature verification keys and tls X.509 certificates diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 9e6eaaa532..25922e5a9c 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -14,6 +14,7 @@ # limitations under the License. from twisted.internet import defer +import six from ._base import SQLBaseStore @@ -21,6 +22,13 @@ from unpaddedbase64 import encode_base64 from synapse.crypto.event_signing import compute_event_reference_hash from synapse.util.caches.descriptors import cached, cachedList +# py2 sqlite has buffer hardcoded as only binary type, so we must use it, +# despite being deprecated and removed in favor of memoryview +if six.PY2: + db_binary_type = buffer +else: + db_binary_type = memoryview + class SignatureWorkerStore(SQLBaseStore): @cached() @@ -56,7 +64,7 @@ class SignatureWorkerStore(SQLBaseStore): for e_id, h in hashes.items() } - defer.returnValue(hashes.items()) + defer.returnValue(list(hashes.items())) def _get_event_reference_hashes_txn(self, txn, event_id): """Get all the hashes for a given PDU. @@ -91,7 +99,7 @@ class SignatureStore(SignatureWorkerStore): vals.append({ "event_id": event.event_id, "algorithm": ref_alg, - "hash": buffer(ref_hash_bytes), + "hash": db_binary_type(ref_hash_bytes), }) self._simple_insert_many_txn( From 7873cde52658d53c3f25ea8257309543af148ab2 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Tue, 29 May 2018 17:35:55 +0200 Subject: [PATCH 273/310] pep8 --- synapse/util/frozenutils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py index 97914907aa..15f0a7ba9e 100644 --- a/synapse/util/frozenutils.py +++ b/synapse/util/frozenutils.py @@ -18,6 +18,7 @@ import simplejson as json from six import string_types + def freeze(o): if isinstance(o, dict): return frozendict({k: freeze(v) for k, v in o.items()}) From 4b9d0cde97495cee16f650f7625c89339a24230c Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Tue, 29 May 2018 17:42:43 +0200 Subject: [PATCH 274/310] add remaining memoryview changes --- synapse/storage/keys.py | 6 +++--- synapse/storage/transactions.py | 10 +++++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index e56c464852..0f13b61da8 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -80,7 +80,7 @@ class KeyStore(SQLBaseStore): values={ "from_server": from_server, "ts_added_ms": time_now_ms, - "tls_certificate": buffer(tls_certificate_bytes), + "tls_certificate": db_binary_type(tls_certificate_bytes), }, desc="store_server_certificate", ) @@ -143,7 +143,7 @@ class KeyStore(SQLBaseStore): values={ "from_server": from_server, "ts_added_ms": time_now_ms, - "verify_key": buffer(verify_key.encode()), + "verify_key": db_binary_type(verify_key.encode()), }, ) txn.call_after( @@ -180,7 +180,7 @@ class KeyStore(SQLBaseStore): "from_server": from_server, "ts_added_ms": ts_now_ms, "ts_valid_until_ms": ts_expires_ms, - "key_json": buffer(key_json_bytes), + "key_json": db_binary_type(key_json_bytes), }, desc="store_server_keys_json", ) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index f825264ea9..e485d19b84 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -17,6 +17,7 @@ from ._base import SQLBaseStore from synapse.util.caches.descriptors import cached from twisted.internet import defer +import six from canonicaljson import encode_canonical_json @@ -25,6 +26,13 @@ from collections import namedtuple import logging import simplejson as json +# py2 sqlite has buffer hardcoded as only binary type, so we must use it, +# despite being deprecated and removed in favor of memoryview +if six.PY2: + db_binary_type = buffer +else: + db_binary_type = memoryview + logger = logging.getLogger(__name__) @@ -110,7 +118,7 @@ class TransactionStore(SQLBaseStore): "transaction_id": transaction_id, "origin": origin, "response_code": code, - "response_json": buffer(encode_canonical_json(response_dict)), + "response_json": db_binary_type(encode_canonical_json(response_dict)), "ts": self._clock.time_msec(), }, or_ignore=True, From c379acd4fd07c41e8181af508093a50e1d894f92 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Tue, 29 May 2018 17:47:28 +0100 Subject: [PATCH 275/310] bump version --- synapse/storage/prepare_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index c08e9cd65a..cf2aae0468 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 49 +SCHEMA_VERSION = 50 dir_path = os.path.abspath(os.path.dirname(__file__)) From 558f3d376a5a6f111bfd277528a510c8157aa99c Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Tue, 29 May 2018 17:47:55 +0100 Subject: [PATCH 276/310] create index in background --- synapse/storage/registration.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index d8e60d2e87..c0bf5e9ed6 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -101,6 +101,13 @@ class RegistrationStore(RegistrationWorkerStore, columns=["user_id", "device_id"], ) + self.register_background_index_update( + "users_creation_ts", + index_name="users_creation_ts", + table="users", + columns=["creation_ts"], + ) + # we no longer use refresh tokens, but it's possible that some people # might have a background update queued to build this index. Just # clear the background update. From ab0ef31dc72313b05ded0e2a3427d278a58ea92d Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Tue, 29 May 2018 17:51:08 +0100 Subject: [PATCH 277/310] create users index on creation_ts --- .../delta/50/add_creation_ts_users_index.sql | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 synapse/storage/schema/delta/50/add_creation_ts_users_index.sql diff --git a/synapse/storage/schema/delta/50/add_creation_ts_users_index.sql b/synapse/storage/schema/delta/50/add_creation_ts_users_index.sql new file mode 100644 index 0000000000..ba33acfd51 --- /dev/null +++ b/synapse/storage/schema/delta/50/add_creation_ts_users_index.sql @@ -0,0 +1,20 @@ +/* Copyright 2018 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +CREATE INDEX users_creation_ts ON users(creation_ts); +INSERT into background_updates (update_name, progress_json) + VALUES ('users_creation_ts', '{}'); From 4a9cbdbc1543378a5f2a5bfde825c3886deaadf5 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 29 May 2018 19:54:32 +0100 Subject: [PATCH 278/310] Exempt AS-registered users from doing gdpr --- synapse/handlers/message.py | 3 +++ synapse/storage/registration.py | 1 + 2 files changed, 4 insertions(+) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 81cff0870e..1cb81b6cf8 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -572,6 +572,9 @@ class EventCreationHandler(object): u = yield self.store.get_user_by_id(user_id) assert u is not None + if u["appservice_id"] is not None: + # users registered by an appservice are exempt + return if u["consent_version"] == self.config.user_consent_version: return diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index a530e29f43..40f7cc16ee 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -36,6 +36,7 @@ class RegistrationWorkerStore(SQLBaseStore): retcols=[ "name", "password_hash", "is_guest", "consent_version", "consent_server_notice_sent", + "appservice_id", ], allow_none=True, desc="get_user_by_id", From a995fdae39303ae2e9ef88e34b7e4e626e23fa73 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 29 May 2018 20:19:29 +0100 Subject: [PATCH 279/310] fix tests --- tests/storage/test_registration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index cc637dda1c..f863b75846 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -49,6 +49,7 @@ class RegistrationStoreTestCase(unittest.TestCase): "is_guest": 0, "consent_version": None, "consent_server_notice_sent": None, + "appservice_id": None, }, (yield self.store.get_user_by_id(self.user_id)) ) From 34b85df7f53763850b358e6e887ebd4f47736b5e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 29 May 2018 22:31:18 +0100 Subject: [PATCH 280/310] Update some comments and docstrings in SyncHandler --- synapse/handlers/sync.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index d0c99c35e3..8377650b68 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -443,6 +443,10 @@ class SyncHandler(object): Returns: A Deferred map from ((type, state_key)->Event) """ + # FIXME this claims to get the state at a stream position, but + # get_recent_events_for_room operates by topo ordering. This therefore + # does not reliably give you the state at the given stream position. + # (https://github.com/matrix-org/synapse/issues/3305) last_events, _ = yield self.store.get_recent_events_for_room( room_id, end_token=stream_position.room_key, limit=1, ) @@ -1042,7 +1046,13 @@ class SyncHandler(object): Returns: Deferred(tuple): Returns a tuple of the form: - `([RoomSyncResultBuilder], [InvitedSyncResult], newly_joined_rooms)` + `(room_entries, invited_rooms, newly_joined_rooms, newly_left_rooms)` + + where: + room_entries is a list [RoomSyncResultBuilder] + invited_rooms is a list [InvitedSyncResult] + newly_joined rooms is a list[str] of room ids + newly_left_rooms is a list[str] of room ids """ user_id = sync_result_builder.sync_config.user.to_string() since_token = sync_result_builder.since_token From c936a52a9eb18e302fbd5158da7188f674912530 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 31 May 2018 19:03:47 +1000 Subject: [PATCH 281/310] Consistently use six's iteritems and wrap lazy keys/values in list() if they're not meant to be lazy (#3307) --- synapse/api/auth.py | 4 ++- synapse/api/filtering.py | 2 +- synapse/event_auth.py | 4 +-- synapse/events/__init__.py | 2 +- synapse/federation/federation_client.py | 4 +-- synapse/federation/send_queue.py | 2 +- synapse/federation/transaction_queue.py | 6 ++-- synapse/handlers/_base.py | 4 +-- synapse/handlers/appservice.py | 4 ++- synapse/handlers/auth.py | 6 ++-- synapse/handlers/device.py | 4 +-- synapse/handlers/federation.py | 17 +++++---- synapse/handlers/presence.py | 12 +++---- synapse/handlers/room.py | 2 +- synapse/handlers/room_list.py | 3 +- synapse/handlers/search.py | 2 +- synapse/handlers/sync.py | 6 ++-- synapse/push/baserules.py | 2 +- synapse/push/mailer.py | 3 +- synapse/push/presentable_names.py | 2 +- synapse/rest/client/transactions.py | 2 +- synapse/state.py | 3 +- synapse/storage/events.py | 47 +++++++++++++------------ synapse/storage/presence.py | 7 ++-- synapse/storage/search.py | 4 +-- synapse/storage/state.py | 47 +++++++++++++------------ synapse/storage/user_directory.py | 8 +++-- synapse/util/caches/descriptors.py | 2 +- synapse/util/caches/treecache.py | 6 ++-- 29 files changed, 116 insertions(+), 101 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index b052cf532b..06fa38366d 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -15,6 +15,8 @@ import logging +from six import itervalues + import pymacaroons from twisted.internet import defer @@ -66,7 +68,7 @@ class Auth(object): ) auth_events = yield self.store.get_events(auth_events_ids) auth_events = { - (e.type, e.state_key): e for e in auth_events.values() + (e.type, e.state_key): e for e in itervalues(auth_events) } self.check(event, auth_events=auth_events, do_sig_check=do_sig_check) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index db43219d24..dbc0e7e445 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -411,7 +411,7 @@ class Filter(object): return room_ids def filter(self, events): - return filter(self.check, events) + return list(filter(self.check, events)) def limit(self): return self.filter_json.get("limit", 10) diff --git a/synapse/event_auth.py b/synapse/event_auth.py index cd5627e36a..eaf9cecde6 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -471,14 +471,14 @@ def _check_power_levels(event, auth_events): ] old_list = current_state.content.get("users", {}) - for user in set(old_list.keys() + user_list.keys()): + for user in set(list(old_list) + list(user_list)): levels_to_check.append( (user, "users") ) old_list = current_state.content.get("events", {}) new_list = event.content.get("events", {}) - for ev_id in set(old_list.keys() + new_list.keys()): + for ev_id in set(list(old_list) + list(new_list)): levels_to_check.append( (ev_id, "events") ) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index c3ff85c49a..cb08da4984 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -146,7 +146,7 @@ class EventBase(object): return field in self._event_dict def items(self): - return self._event_dict.items() + return list(self._event_dict.items()) class FrozenEvent(EventBase): diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 2761ffae07..87a92f6ea9 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -391,7 +391,7 @@ class FederationClient(FederationBase): """ if return_local: seen_events = yield self.store.get_events(event_ids, allow_rejected=True) - signed_events = seen_events.values() + signed_events = list(seen_events.values()) else: seen_events = yield self.store.have_seen_events(event_ids) signed_events = [] @@ -589,7 +589,7 @@ class FederationClient(FederationBase): } valid_pdus = yield self._check_sigs_and_hash_and_fetch( - destination, pdus.values(), + destination, list(pdus.values()), outlier=True, ) diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py index c7ed465617..3dcc629d44 100644 --- a/synapse/federation/send_queue.py +++ b/synapse/federation/send_queue.py @@ -197,7 +197,7 @@ class FederationRemoteSendQueue(object): # We only want to send presence for our own users, so lets always just # filter here just in case. - local_states = filter(lambda s: self.is_mine_id(s.user_id), states) + local_states = list(filter(lambda s: self.is_mine_id(s.user_id), states)) self.presence_map.update({state.user_id: state for state in local_states}) self.presence_changed[pos] = [state.user_id for state in local_states] diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 69312ec233..f0aeb5a0d3 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -35,6 +35,8 @@ from synapse.metrics import ( from prometheus_client import Counter +from six import itervalues + import logging @@ -234,7 +236,7 @@ class TransactionQueue(object): yield logcontext.make_deferred_yieldable(defer.gatherResults( [ logcontext.run_in_background(handle_room_events, evs) - for evs in events_by_room.itervalues() + for evs in itervalues(events_by_room) ], consumeErrors=True )) @@ -325,7 +327,7 @@ class TransactionQueue(object): if not states_map: break - yield self._process_presence_inner(states_map.values()) + yield self._process_presence_inner(list(states_map.values())) except Exception: logger.exception("Error sending presence states to servers") finally: diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index e089e66fde..2d1db0c245 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -114,14 +114,14 @@ class BaseHandler(object): if guest_access != "can_join": if context: current_state = yield self.store.get_events( - context.current_state_ids.values() + list(context.current_state_ids.values()) ) else: current_state = yield self.state_handler.get_current_state( event.room_id ) - current_state = current_state.values() + current_state = list(current_state.values()) logger.info("maybe_kick_guest_users %r", current_state) yield self.kick_guest_users(current_state) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index d9f35a5dba..1c29c43a83 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -15,6 +15,8 @@ from twisted.internet import defer +from six import itervalues + import synapse from synapse.api.constants import EventTypes from synapse.util.metrics import Measure @@ -119,7 +121,7 @@ class ApplicationServicesHandler(object): yield make_deferred_yieldable(defer.gatherResults([ run_in_background(handle_room_events, evs) - for evs in events_by_room.itervalues() + for evs in itervalues(events_by_room) ], consumeErrors=True)) yield self.store.set_appservice_last_pos(upper_bound) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index a5365c4fe4..3c0051586d 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -249,7 +249,7 @@ class AuthHandler(BaseHandler): errordict = e.error_dict() for f in flows: - if len(set(f) - set(creds.keys())) == 0: + if len(set(f) - set(creds)) == 0: # it's very useful to know what args are stored, but this can # include the password in the case of registering, so only log # the keys (confusingly, clientdict may contain a password @@ -257,12 +257,12 @@ class AuthHandler(BaseHandler): # and is not sensitive). logger.info( "Auth completed with creds: %r. Client dict has keys: %r", - creds, clientdict.keys() + creds, list(clientdict) ) defer.returnValue((creds, clientdict, session['id'])) ret = self._auth_dict_for_flows(flows, session) - ret['completed'] = creds.keys() + ret['completed'] = list(creds) ret.update(errordict) raise InteractiveAuthIncompleteError( ret, diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 31bd0e60c6..11c6fb3657 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -114,7 +114,7 @@ class DeviceHandler(BaseHandler): user_id, device_id=None ) - devices = device_map.values() + devices = list(device_map.values()) for device in devices: _update_device_from_client_ips(device, ips) @@ -187,7 +187,7 @@ class DeviceHandler(BaseHandler): defer.Deferred: """ device_map = yield self.store.get_devices_by_user(user_id) - device_ids = device_map.keys() + device_ids = list(device_map) if except_device_id is not None: device_ids = [d for d in device_ids if d != except_device_id] yield self.delete_devices(user_id, device_ids) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 87c0615820..fcf94befb7 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -52,7 +52,6 @@ from synapse.util.retryutils import NotRetryingDestination from synapse.util.distributor import user_joined_room - logger = logging.getLogger(__name__) @@ -480,8 +479,8 @@ class FederationHandler(BaseHandler): # to get all state ids that we're interested in. event_map = yield self.store.get_events([ e_id - for key_to_eid in event_to_state_ids.itervalues() - for key, e_id in key_to_eid.iteritems() + for key_to_eid in list(event_to_state_ids.values()) + for key, e_id in key_to_eid.items() if key[0] != EventTypes.Member or check_match(key[1]) ]) @@ -1149,13 +1148,13 @@ class FederationHandler(BaseHandler): user = UserID.from_string(event.state_key) yield user_joined_room(self.distributor, user, event.room_id) - state_ids = context.prev_state_ids.values() + state_ids = list(context.prev_state_ids.values()) auth_chain = yield self.store.get_auth_chain(state_ids) - state = yield self.store.get_events(context.prev_state_ids.values()) + state = yield self.store.get_events(list(context.prev_state_ids.values())) defer.returnValue({ - "state": state.values(), + "state": list(state.values()), "auth_chain": auth_chain, }) @@ -1405,7 +1404,7 @@ class FederationHandler(BaseHandler): else: del results[(event.type, event.state_key)] - res = results.values() + res = list(results.values()) for event in res: # We sign these again because there was a bug where we # incorrectly signed things the first time round @@ -1446,7 +1445,7 @@ class FederationHandler(BaseHandler): else: results.pop((event.type, event.state_key), None) - defer.returnValue(results.values()) + defer.returnValue(list(results.values())) else: defer.returnValue([]) @@ -1915,7 +1914,7 @@ class FederationHandler(BaseHandler): }) new_state = self.state_handler.resolve_events( - [local_view.values(), remote_view.values()], + [list(local_view.values()), list(remote_view.values())], event ) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 26fc0d3ec7..7fe568132f 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -325,7 +325,7 @@ class PresenceHandler(object): if to_notify: notified_presence_counter.inc(len(to_notify)) - yield self._persist_and_notify(to_notify.values()) + yield self._persist_and_notify(list(to_notify.values())) self.unpersisted_users_changes |= set(s.user_id for s in new_states) self.unpersisted_users_changes -= set(to_notify.keys()) @@ -687,7 +687,7 @@ class PresenceHandler(object): """ updates = yield self.current_state_for_users(target_user_ids) - updates = updates.values() + updates = list(updates.values()) for user_id in set(target_user_ids) - set(u.user_id for u in updates): updates.append(UserPresenceState.default(user_id)) @@ -753,11 +753,11 @@ class PresenceHandler(object): self._push_to_remotes([state]) else: user_ids = yield self.store.get_users_in_room(room_id) - user_ids = filter(self.is_mine_id, user_ids) + user_ids = list(filter(self.is_mine_id, user_ids)) states = yield self.current_state_for_users(user_ids) - self._push_to_remotes(states.values()) + self._push_to_remotes(list(states.values())) @defer.inlineCallbacks def get_presence_list(self, observer_user, accepted=None): @@ -1051,7 +1051,7 @@ class PresenceEventSource(object): updates = yield presence.current_state_for_users(user_ids_changed) if include_offline: - defer.returnValue((updates.values(), max_token)) + defer.returnValue((list(updates.values()), max_token)) else: defer.returnValue(([ s for s in itervalues(updates) @@ -1112,7 +1112,7 @@ def handle_timeouts(user_states, is_mine_fn, syncing_user_ids, now): if new_state: changes[state.user_id] = new_state - return changes.values() + return list(changes.values()) def handle_timeout(state, is_mine, syncing_user_ids, now): diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index b5850db42f..2abd63ad05 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -455,7 +455,7 @@ class RoomContextHandler(BaseHandler): state = yield self.store.get_state_for_events( [last_event_id], None ) - results["state"] = state[last_event_id].values() + results["state"] = list(state[last_event_id].values()) results["start"] = now_token.copy_and_replace( "room_key", results["start"] diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 5757bb7f8a..fc507cef36 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -15,6 +15,7 @@ from twisted.internet import defer +from six import iteritems from six.moves import range from ._base import BaseHandler @@ -307,7 +308,7 @@ class RoomListHandler(BaseHandler): ) event_map = yield self.store.get_events([ - event_id for key, event_id in current_state_ids.iteritems() + event_id for key, event_id in iteritems(current_state_ids) if key[0] in ( EventTypes.JoinRules, EventTypes.Name, diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 9772ed1a0e..1eca26aa1e 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -348,7 +348,7 @@ class SearchHandler(BaseHandler): rooms = set(e.room_id for e in allowed_events) for room_id in rooms: state = yield self.state_handler.get_current_state(room_id) - state_results[room_id] = state.values() + state_results[room_id] = list(state.values()) state_results.values() diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 8377650b68..51ec727df0 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -541,11 +541,11 @@ class SyncHandler(object): state = {} if state_ids: - state = yield self.store.get_events(state_ids.values()) + state = yield self.store.get_events(list(state_ids.values())) defer.returnValue({ (e.type, e.state_key): e - for e in sync_config.filter_collection.filter_room_state(state.values()) + for e in sync_config.filter_collection.filter_room_state(list(state.values())) }) @defer.inlineCallbacks @@ -894,7 +894,7 @@ class SyncHandler(object): presence.extend(states) # Deduplicate the presence entries so that there's at most one per user - presence = {p.user_id: p for p in presence}.values() + presence = list({p.user_id: p for p in presence}.values()) presence = sync_config.filter_collection.filter_presence( presence diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 7a18afe5f9..a8ae7bcd6c 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -39,7 +39,7 @@ def list_with_base_rules(rawrules): rawrules = [r for r in rawrules if r['priority_class'] >= 0] # shove the server default rules for each kind onto the end of each - current_prio_class = PRIORITY_CLASS_INVERSE_MAP.keys()[-1] + current_prio_class = list(PRIORITY_CLASS_INVERSE_MAP)[-1] ruleslist.extend(make_base_prepend_rules( PRIORITY_CLASS_INVERSE_MAP[current_prio_class], modified_base_rules diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index b5cd9b426a..d4be800e5e 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -229,7 +229,8 @@ class Mailer(object): if room_vars['notifs'] and 'messages' in room_vars['notifs'][-1]: prev_messages = room_vars['notifs'][-1]['messages'] for message in notifvars['messages']: - pm = filter(lambda pm: pm['id'] == message['id'], prev_messages) + pm = list(filter(lambda pm: pm['id'] == message['id'], + prev_messages)) if pm: if not message["is_historical"]: pm[0]["is_historical"] = False diff --git a/synapse/push/presentable_names.py b/synapse/push/presentable_names.py index 277da3cd35..43f0c74ff3 100644 --- a/synapse/push/presentable_names.py +++ b/synapse/push/presentable_names.py @@ -113,7 +113,7 @@ def calculate_room_name(store, room_state_ids, user_id, fallback_to_members=True # so find out who is in the room that isn't the user. if "m.room.member" in room_state_bytype_ids: member_events = yield store.get_events( - room_state_bytype_ids["m.room.member"].values() + list(room_state_bytype_ids["m.room.member"].values()) ) all_members = [ ev for ev in member_events.values() diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py index 20fa6678ef..7c01b438cb 100644 --- a/synapse/rest/client/transactions.py +++ b/synapse/rest/client/transactions.py @@ -104,7 +104,7 @@ class HttpTransactionCache(object): def _cleanup(self): now = self.clock.time_msec() - for key in self.transactions.keys(): + for key in list(self.transactions): ts = self.transactions[key][1] if now > (ts + CLEANUP_PERIOD_MS): # after cleanup period del self.transactions[key] diff --git a/synapse/state.py b/synapse/state.py index b8c27c6815..216418f58d 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -132,7 +132,8 @@ class StateHandler(object): defer.returnValue(event) return - state_map = yield self.store.get_events(state.values(), get_prev_content=False) + state_map = yield self.store.get_events(list(state.values()), + get_prev_content=False) state = { key: state_map[e_id] for key, e_id in iteritems(state) if e_id in state_map } diff --git a/synapse/storage/events.py b/synapse/storage/events.py index b96104ccae..cb1082e864 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -40,6 +40,9 @@ import synapse.metrics from synapse.events import EventBase # noqa: F401 from synapse.events.snapshot import EventContext # noqa: F401 +from six.moves import range +from six import itervalues, iteritems + from prometheus_client import Counter logger = logging.getLogger(__name__) @@ -245,7 +248,7 @@ class EventsStore(EventsWorkerStore): partitioned.setdefault(event.room_id, []).append((event, ctx)) deferreds = [] - for room_id, evs_ctxs in partitioned.iteritems(): + for room_id, evs_ctxs in iteritems(partitioned): d = self._event_persist_queue.add_to_queue( room_id, evs_ctxs, backfilled=backfilled, @@ -330,7 +333,7 @@ class EventsStore(EventsWorkerStore): chunks = [ events_and_contexts[x:x + 100] - for x in xrange(0, len(events_and_contexts), 100) + for x in range(0, len(events_and_contexts), 100) ] for chunk in chunks: @@ -364,7 +367,7 @@ class EventsStore(EventsWorkerStore): (event, context) ) - for room_id, ev_ctx_rm in events_by_room.iteritems(): + for room_id, ev_ctx_rm in iteritems(events_by_room): # Work out new extremities by recursively adding and removing # the new events. latest_event_ids = yield self.get_latest_event_ids_in_room( @@ -459,12 +462,12 @@ class EventsStore(EventsWorkerStore): event_counter.labels(event.type, origin_type, origin_entity).inc() - for room_id, new_state in current_state_for_room.iteritems(): + for room_id, new_state in iteritems(current_state_for_room): self.get_current_state_ids.prefill( (room_id, ), new_state ) - for room_id, latest_event_ids in new_forward_extremeties.iteritems(): + for room_id, latest_event_ids in iteritems(new_forward_extremeties): self.get_latest_event_ids_in_room.prefill( (room_id,), list(latest_event_ids) ) @@ -641,20 +644,20 @@ class EventsStore(EventsWorkerStore): """ existing_state = yield self.get_current_state_ids(room_id) - existing_events = set(existing_state.itervalues()) - new_events = set(ev_id for ev_id in current_state.itervalues()) + existing_events = set(itervalues(existing_state)) + new_events = set(ev_id for ev_id in itervalues(current_state)) changed_events = existing_events ^ new_events if not changed_events: return to_delete = { - key: ev_id for key, ev_id in existing_state.iteritems() + key: ev_id for key, ev_id in iteritems(existing_state) if ev_id in changed_events } events_to_insert = (new_events - existing_events) to_insert = { - key: ev_id for key, ev_id in current_state.iteritems() + key: ev_id for key, ev_id in iteritems(current_state) if ev_id in events_to_insert } @@ -757,11 +760,11 @@ class EventsStore(EventsWorkerStore): ) def _update_current_state_txn(self, txn, state_delta_by_room, max_stream_order): - for room_id, current_state_tuple in state_delta_by_room.iteritems(): + for room_id, current_state_tuple in iteritems(state_delta_by_room): to_delete, to_insert = current_state_tuple txn.executemany( "DELETE FROM current_state_events WHERE event_id = ?", - [(ev_id,) for ev_id in to_delete.itervalues()], + [(ev_id,) for ev_id in itervalues(to_delete)], ) self._simple_insert_many_txn( @@ -774,7 +777,7 @@ class EventsStore(EventsWorkerStore): "type": key[0], "state_key": key[1], } - for key, ev_id in to_insert.iteritems() + for key, ev_id in iteritems(to_insert) ], ) @@ -793,7 +796,7 @@ class EventsStore(EventsWorkerStore): "event_id": ev_id, "prev_event_id": to_delete.get(key, None), } - for key, ev_id in state_deltas.iteritems() + for key, ev_id in iteritems(state_deltas) ] ) @@ -836,7 +839,7 @@ class EventsStore(EventsWorkerStore): def _update_forward_extremities_txn(self, txn, new_forward_extremities, max_stream_order): - for room_id, new_extrem in new_forward_extremities.iteritems(): + for room_id, new_extrem in iteritems(new_forward_extremities): self._simple_delete_txn( txn, table="event_forward_extremities", @@ -854,7 +857,7 @@ class EventsStore(EventsWorkerStore): "event_id": ev_id, "room_id": room_id, } - for room_id, new_extrem in new_forward_extremities.iteritems() + for room_id, new_extrem in iteritems(new_forward_extremities) for ev_id in new_extrem ], ) @@ -871,7 +874,7 @@ class EventsStore(EventsWorkerStore): "event_id": event_id, "stream_ordering": max_stream_order, } - for room_id, new_extrem in new_forward_extremities.iteritems() + for room_id, new_extrem in iteritems(new_forward_extremities) for event_id in new_extrem ] ) @@ -899,7 +902,7 @@ class EventsStore(EventsWorkerStore): new_events_and_contexts[event.event_id] = (event, context) else: new_events_and_contexts[event.event_id] = (event, context) - return new_events_and_contexts.values() + return list(new_events_and_contexts.values()) def _update_room_depths_txn(self, txn, events_and_contexts, backfilled): """Update min_depth for each room @@ -925,7 +928,7 @@ class EventsStore(EventsWorkerStore): event.depth, depth_updates.get(event.room_id, event.depth) ) - for room_id, depth in depth_updates.iteritems(): + for room_id, depth in iteritems(depth_updates): self._update_min_depth_for_room_txn(txn, room_id, depth) def _update_outliers_txn(self, txn, events_and_contexts): @@ -1309,7 +1312,7 @@ class EventsStore(EventsWorkerStore): " WHERE e.event_id IN (%s)" ) % (",".join(["?"] * len(ev_map)),) - txn.execute(sql, ev_map.keys()) + txn.execute(sql, list(ev_map)) rows = self.cursor_to_dict(txn) for row in rows: event = ev_map[row["event_id"]] @@ -1572,7 +1575,7 @@ class EventsStore(EventsWorkerStore): chunks = [ event_ids[i:i + 100] - for i in xrange(0, len(event_ids), 100) + for i in range(0, len(event_ids), 100) ] for chunk in chunks: ev_rows = self._simple_select_many_txn( @@ -1986,7 +1989,7 @@ class EventsStore(EventsWorkerStore): logger.info("[purge] finding state groups which depend on redundant" " state groups") remaining_state_groups = [] - for i in xrange(0, len(state_rows), 100): + for i in range(0, len(state_rows), 100): chunk = [sg for sg, in state_rows[i:i + 100]] # look for state groups whose prev_state_group is one we are about # to delete @@ -2042,7 +2045,7 @@ class EventsStore(EventsWorkerStore): "state_key": key[1], "event_id": state_id, } - for key, state_id in curr_state.iteritems() + for key, state_id in iteritems(curr_state) ], ) diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index 9e9d3c2591..f05d91cc58 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -16,6 +16,7 @@ from ._base import SQLBaseStore from synapse.api.constants import PresenceState from synapse.util.caches.descriptors import cached, cachedInlineCallbacks, cachedList +from synapse.util import batch_iter from collections import namedtuple from twisted.internet import defer @@ -115,11 +116,7 @@ class PresenceStore(SQLBaseStore): " AND user_id IN (%s)" ) - batches = ( - presence_states[i:i + 50] - for i in xrange(0, len(presence_states), 50) - ) - for states in batches: + for states in batch_iter(presence_states, 50): args = [stream_id] args.extend(s.user_id for s in states) txn.execute( diff --git a/synapse/storage/search.py b/synapse/storage/search.py index a9c299a861..f0fa5d7631 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -448,7 +448,7 @@ class SearchStore(BackgroundUpdateStore): "search_msgs", self.cursor_to_dict, sql, *args ) - results = filter(lambda row: row["room_id"] in room_ids, results) + results = list(filter(lambda row: row["room_id"] in room_ids, results)) events = yield self._get_events([r["event_id"] for r in results]) @@ -603,7 +603,7 @@ class SearchStore(BackgroundUpdateStore): "search_rooms", self.cursor_to_dict, sql, *args ) - results = filter(lambda row: row["room_id"] in room_ids, results) + results = list(filter(lambda row: row["room_id"] in room_ids, results)) events = yield self._get_events([r["event_id"] for r in results]) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index ffa4246031..bdee14a8eb 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -16,6 +16,9 @@ from collections import namedtuple import logging +from six import iteritems, itervalues +from six.moves import range + from twisted.internet import defer from synapse.storage.background_updates import BackgroundUpdateStore @@ -134,7 +137,7 @@ class StateGroupWorkerStore(SQLBaseStore): event_ids, ) - groups = set(event_to_groups.itervalues()) + groups = set(itervalues(event_to_groups)) group_to_state = yield self._get_state_for_groups(groups) defer.returnValue(group_to_state) @@ -166,18 +169,18 @@ class StateGroupWorkerStore(SQLBaseStore): state_event_map = yield self.get_events( [ - ev_id for group_ids in group_to_ids.itervalues() - for ev_id in group_ids.itervalues() + ev_id for group_ids in itervalues(group_to_ids) + for ev_id in itervalues(group_ids) ], get_prev_content=False ) defer.returnValue({ group: [ - state_event_map[v] for v in event_id_map.itervalues() + state_event_map[v] for v in itervalues(event_id_map) if v in state_event_map ] - for group, event_id_map in group_to_ids.iteritems() + for group, event_id_map in iteritems(group_to_ids) }) @defer.inlineCallbacks @@ -186,7 +189,7 @@ class StateGroupWorkerStore(SQLBaseStore): """ results = {} - chunks = [groups[i:i + 100] for i in xrange(0, len(groups), 100)] + chunks = [groups[i:i + 100] for i in range(0, len(groups), 100)] for chunk in chunks: res = yield self.runInteraction( "_get_state_groups_from_groups", @@ -347,21 +350,21 @@ class StateGroupWorkerStore(SQLBaseStore): event_ids, ) - groups = set(event_to_groups.itervalues()) + groups = set(itervalues(event_to_groups)) group_to_state = yield self._get_state_for_groups(groups, types) state_event_map = yield self.get_events( - [ev_id for sd in group_to_state.itervalues() for ev_id in sd.itervalues()], + [ev_id for sd in itervalues(group_to_state) for ev_id in itervalues(sd)], get_prev_content=False ) event_to_state = { event_id: { k: state_event_map[v] - for k, v in group_to_state[group].iteritems() + for k, v in iteritems(group_to_state[group]) if v in state_event_map } - for event_id, group in event_to_groups.iteritems() + for event_id, group in iteritems(event_to_groups) } defer.returnValue({event: event_to_state[event] for event in event_ids}) @@ -384,12 +387,12 @@ class StateGroupWorkerStore(SQLBaseStore): event_ids, ) - groups = set(event_to_groups.itervalues()) + groups = set(itervalues(event_to_groups)) group_to_state = yield self._get_state_for_groups(groups, types) event_to_state = { event_id: group_to_state[group] - for event_id, group in event_to_groups.iteritems() + for event_id, group in iteritems(event_to_groups) } defer.returnValue({event: event_to_state[event] for event in event_ids}) @@ -503,7 +506,7 @@ class StateGroupWorkerStore(SQLBaseStore): got_all = is_all or not missing_types return { - k: v for k, v in state_dict_ids.iteritems() + k: v for k, v in iteritems(state_dict_ids) if include(k[0], k[1]) }, missing_types, got_all @@ -562,12 +565,12 @@ class StateGroupWorkerStore(SQLBaseStore): # Now we want to update the cache with all the things we fetched # from the database. - for group, group_state_dict in group_to_state_dict.iteritems(): + for group, group_state_dict in iteritems(group_to_state_dict): state_dict = results[group] state_dict.update( ((intern_string(k[0]), intern_string(k[1])), to_ascii(v)) - for k, v in group_state_dict.iteritems() + for k, v in iteritems(group_state_dict) ) self._state_group_cache.update( @@ -654,7 +657,7 @@ class StateGroupWorkerStore(SQLBaseStore): "state_key": key[1], "event_id": state_id, } - for key, state_id in delta_ids.iteritems() + for key, state_id in iteritems(delta_ids) ], ) else: @@ -669,7 +672,7 @@ class StateGroupWorkerStore(SQLBaseStore): "state_key": key[1], "event_id": state_id, } - for key, state_id in current_state_ids.iteritems() + for key, state_id in iteritems(current_state_ids) ], ) @@ -794,11 +797,11 @@ class StateStore(StateGroupWorkerStore, BackgroundUpdateStore): "state_group": state_group_id, "event_id": event_id, } - for event_id, state_group_id in state_groups.iteritems() + for event_id, state_group_id in iteritems(state_groups) ], ) - for event_id, state_group_id in state_groups.iteritems(): + for event_id, state_group_id in iteritems(state_groups): txn.call_after( self._get_state_group_for_event.prefill, (event_id,), state_group_id @@ -826,7 +829,7 @@ class StateStore(StateGroupWorkerStore, BackgroundUpdateStore): def reindex_txn(txn): new_last_state_group = last_state_group - for count in xrange(batch_size): + for count in range(batch_size): txn.execute( "SELECT id, room_id FROM state_groups" " WHERE ? < id AND id <= ?" @@ -884,7 +887,7 @@ class StateStore(StateGroupWorkerStore, BackgroundUpdateStore): # of keys delta_state = { - key: value for key, value in curr_state.iteritems() + key: value for key, value in iteritems(curr_state) if prev_state.get(key, None) != value } @@ -924,7 +927,7 @@ class StateStore(StateGroupWorkerStore, BackgroundUpdateStore): "state_key": key[1], "event_id": state_id, } - for key, state_id in delta_state.iteritems() + for key, state_id in iteritems(delta_state) ], ) diff --git a/synapse/storage/user_directory.py b/synapse/storage/user_directory.py index d6e289ffbe..275c299998 100644 --- a/synapse/storage/user_directory.py +++ b/synapse/storage/user_directory.py @@ -22,6 +22,8 @@ from synapse.api.constants import EventTypes, JoinRules from synapse.storage.engines import PostgresEngine, Sqlite3Engine from synapse.types import get_domain_from_id, get_localpart_from_id +from six import iteritems + import re import logging @@ -100,7 +102,7 @@ class UserDirectoryStore(SQLBaseStore): user_id, get_localpart_from_id(user_id), get_domain_from_id(user_id), profile.display_name, ) - for user_id, profile in users_with_profile.iteritems() + for user_id, profile in iteritems(users_with_profile) ) elif isinstance(self.database_engine, Sqlite3Engine): sql = """ @@ -112,7 +114,7 @@ class UserDirectoryStore(SQLBaseStore): user_id, "%s %s" % (user_id, p.display_name,) if p.display_name else user_id ) - for user_id, p in users_with_profile.iteritems() + for user_id, p in iteritems(users_with_profile) ) else: # This should be unreachable. @@ -130,7 +132,7 @@ class UserDirectoryStore(SQLBaseStore): "display_name": profile.display_name, "avatar_url": profile.avatar_url, } - for user_id, profile in users_with_profile.iteritems() + for user_id, profile in iteritems(users_with_profile) ] ) for user_id in users_with_profile: diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py index f4e2c30088..fc1874b65b 100644 --- a/synapse/util/caches/descriptors.py +++ b/synapse/util/caches/descriptors.py @@ -569,7 +569,7 @@ class CacheListDescriptor(_CacheDescriptorBase): return results return logcontext.make_deferred_yieldable(defer.gatherResults( - cached_defers.values(), + list(cached_defers.values()), consumeErrors=True, ).addCallback(update_results_dict).addErrback( unwrapFirstError diff --git a/synapse/util/caches/treecache.py b/synapse/util/caches/treecache.py index fcc341a6b7..dd4c9e6067 100644 --- a/synapse/util/caches/treecache.py +++ b/synapse/util/caches/treecache.py @@ -1,3 +1,5 @@ +from six import itervalues + SENTINEL = object() @@ -49,7 +51,7 @@ class TreeCache(object): if popped is SENTINEL: return default - node_and_keys = zip(nodes, key) + node_and_keys = list(zip(nodes, key)) node_and_keys.reverse() node_and_keys.append((self.root, None)) @@ -76,7 +78,7 @@ def iterate_tree_cache_entry(d): can contain dicts. """ if isinstance(d, dict): - for value_d in d.itervalues(): + for value_d in itervalues(d): for value in iterate_tree_cache_entry(value_d): yield value else: From febe0ec8fd78028fe7c7b3a26a8dd85c32ee1550 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 31 May 2018 19:04:50 +1000 Subject: [PATCH 282/310] Run Prometheus on a different port, optionally. (#3274) --- docs/metrics-howto.rst | 77 +++++++++++++++++++++++++++----- synapse/app/_base.py | 13 ++++++ synapse/app/appservice.py | 7 +++ synapse/app/client_reader.py | 11 ++++- synapse/app/event_creator.py | 10 ++++- synapse/app/federation_reader.py | 10 ++++- synapse/app/federation_sender.py | 10 ++++- synapse/app/frontend_proxy.py | 10 ++++- synapse/app/homeserver.py | 13 ++++-- synapse/app/media_repository.py | 10 ++++- synapse/app/pusher.py | 10 ++++- synapse/app/synchrotron.py | 10 ++++- synapse/app/user_dir.py | 10 ++++- synapse/config/server.py | 10 +++++ synapse/metrics/__init__.py | 3 +- synapse/metrics/resource.py | 4 ++ 16 files changed, 192 insertions(+), 26 deletions(-) diff --git a/docs/metrics-howto.rst b/docs/metrics-howto.rst index 8acc479bc3..25e06bca58 100644 --- a/docs/metrics-howto.rst +++ b/docs/metrics-howto.rst @@ -1,25 +1,47 @@ How to monitor Synapse metrics using Prometheus =============================================== -1. Install prometheus: +1. Install Prometheus: Follow instructions at http://prometheus.io/docs/introduction/install/ -2. Enable synapse metrics: +2. Enable Synapse metrics: - Simply setting a (local) port number will enable it. Pick a port. - prometheus itself defaults to 9090, so starting just above that for - locally monitored services seems reasonable. E.g. 9092: + There are two methods of enabling metrics in Synapse. - Add to homeserver.yaml:: + The first serves the metrics as a part of the usual web server and can be + enabled by adding the "metrics" resource to the existing listener as such:: - metrics_port: 9092 + resources: + - names: + - client + - metrics - Also ensure that ``enable_metrics`` is set to ``True``. + This provides a simple way of adding metrics to your Synapse installation, + and serves under ``/_synapse/metrics``. If you do not wish your metrics be + publicly exposed, you will need to either filter it out at your load + balancer, or use the second method. - Restart synapse. + The second method runs the metrics server on a different port, in a + different thread to Synapse. This can make it more resilient to heavy load + meaning metrics cannot be retrieved, and can be exposed to just internal + networks easier. The served metrics are available over HTTP only, and will + be available at ``/``. -3. Add a prometheus target for synapse. + Add a new listener to homeserver.yaml:: + + listeners: + - type: metrics + port: 9000 + bind_addresses: + - '0.0.0.0' + + For both options, you will need to ensure that ``enable_metrics`` is set to + ``True``. + + Restart Synapse. + +3. Add a Prometheus target for Synapse. It needs to set the ``metrics_path`` to a non-default value (under ``scrape_configs``):: @@ -31,7 +53,40 @@ How to monitor Synapse metrics using Prometheus If your prometheus is older than 1.5.2, you will need to replace ``static_configs`` in the above with ``target_groups``. - Restart prometheus. + Restart Prometheus. + + +Removal of deprecated metrics & time based counters becoming histograms in 0.31.0 +--------------------------------------------------------------------------------- + +The duplicated metrics deprecated in Synapse 0.27.0 have been removed. + +All time duration-based metrics have been changed to be seconds. This affects: + +================================ +msec -> sec metrics +================================ +python_gc_time +python_twisted_reactor_tick_time +synapse_storage_query_time +synapse_storage_schedule_time +synapse_storage_transaction_time +================================ + +Several metrics have been changed to be histograms, which sort entries into +buckets and allow better analysis. The following metrics are now histograms: + +========================================= +Altered metrics +========================================= +python_gc_time +python_twisted_reactor_pending_calls +python_twisted_reactor_tick_time +synapse_http_server_response_time_seconds +synapse_storage_query_time +synapse_storage_schedule_time +synapse_storage_transaction_time +========================================= Block and response metrics renamed for 0.27.0 diff --git a/synapse/app/_base.py b/synapse/app/_base.py index e4318cdfc3..a6925ab139 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -124,6 +124,19 @@ def quit_with_error(error_string): sys.exit(1) +def listen_metrics(bind_addresses, port): + """ + Start Prometheus metrics server. + """ + from synapse.metrics import RegistryProxy + from prometheus_client import start_http_server + + for host in bind_addresses: + reactor.callInThread(start_http_server, int(port), + addr=host, registry=RegistryProxy) + logger.info("Metrics now reporting on %s:%d", host, port) + + def listen_tcp(bind_addresses, port, factory, backlog=50): """ Create a TCP socket for a port and several addresses diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py index b1efacc9f8..dd114dee07 100644 --- a/synapse/app/appservice.py +++ b/synapse/app/appservice.py @@ -94,6 +94,13 @@ class AppserviceServer(HomeServer): globals={"hs": self}, ) ) + elif listener["type"] == "metrics": + if not self.get_config().enable_metrics: + logger.warn(("Metrics listener configured, but " + "collect_metrics is not enabled!")) + else: + _base.listen_metrics(listener["bind_addresses"], + listener["port"]) else: logger.warn("Unrecognized listener type: %s", listener["type"]) diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py index 38b98382c6..85dada7f9f 100644 --- a/synapse/app/client_reader.py +++ b/synapse/app/client_reader.py @@ -25,6 +25,7 @@ from synapse.config.logger import setup_logging from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.site import SynapseSite +from synapse.metrics import RegistryProxy from synapse.metrics.resource import METRICS_PREFIX, MetricsResource from synapse.replication.slave.storage._base import BaseSlavedStore from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore @@ -77,7 +78,7 @@ class ClientReaderServer(HomeServer): for res in listener_config["resources"]: for name in res["names"]: if name == "metrics": - resources[METRICS_PREFIX] = MetricsResource(self) + resources[METRICS_PREFIX] = MetricsResource(RegistryProxy) elif name == "client": resource = JsonResource(self, canonical_json=False) PublicRoomListRestServlet(self).register(resource) @@ -118,7 +119,13 @@ class ClientReaderServer(HomeServer): globals={"hs": self}, ) ) - + elif listener["type"] == "metrics": + if not self.get_config().enable_metrics: + logger.warn(("Metrics listener configured, but " + "collect_metrics is not enabled!")) + else: + _base.listen_metrics(listener["bind_addresses"], + listener["port"]) else: logger.warn("Unrecognized listener type: %s", listener["type"]) diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py index bd7f3d5679..5ca77c0f1a 100644 --- a/synapse/app/event_creator.py +++ b/synapse/app/event_creator.py @@ -25,6 +25,7 @@ from synapse.config.logger import setup_logging from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.site import SynapseSite +from synapse.metrics import RegistryProxy from synapse.metrics.resource import METRICS_PREFIX, MetricsResource from synapse.replication.slave.storage._base import BaseSlavedStore from synapse.replication.slave.storage.account_data import SlavedAccountDataStore @@ -90,7 +91,7 @@ class EventCreatorServer(HomeServer): for res in listener_config["resources"]: for name in res["names"]: if name == "metrics": - resources[METRICS_PREFIX] = MetricsResource(self) + resources[METRICS_PREFIX] = MetricsResource(RegistryProxy) elif name == "client": resource = JsonResource(self, canonical_json=False) RoomSendEventRestServlet(self).register(resource) @@ -134,6 +135,13 @@ class EventCreatorServer(HomeServer): globals={"hs": self}, ) ) + elif listener["type"] == "metrics": + if not self.get_config().enable_metrics: + logger.warn(("Metrics listener configured, but " + "collect_metrics is not enabled!")) + else: + _base.listen_metrics(listener["bind_addresses"], + listener["port"]) else: logger.warn("Unrecognized listener type: %s", listener["type"]) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index 6e10b27b9e..2a1995d0cd 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -26,6 +26,7 @@ from synapse.config.logger import setup_logging from synapse.crypto import context_factory from synapse.federation.transport.server import TransportLayerServer from synapse.http.site import SynapseSite +from synapse.metrics import RegistryProxy from synapse.metrics.resource import METRICS_PREFIX, MetricsResource from synapse.replication.slave.storage._base import BaseSlavedStore from synapse.replication.slave.storage.directory import DirectoryStore @@ -71,7 +72,7 @@ class FederationReaderServer(HomeServer): for res in listener_config["resources"]: for name in res["names"]: if name == "metrics": - resources[METRICS_PREFIX] = MetricsResource(self) + resources[METRICS_PREFIX] = MetricsResource(RegistryProxy) elif name == "federation": resources.update({ FEDERATION_PREFIX: TransportLayerServer(self), @@ -107,6 +108,13 @@ class FederationReaderServer(HomeServer): globals={"hs": self}, ) ) + elif listener["type"] == "metrics": + if not self.get_config().enable_metrics: + logger.warn(("Metrics listener configured, but " + "collect_metrics is not enabled!")) + else: + _base.listen_metrics(listener["bind_addresses"], + listener["port"]) else: logger.warn("Unrecognized listener type: %s", listener["type"]) diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py index 6f24e32d6d..81ad574043 100644 --- a/synapse/app/federation_sender.py +++ b/synapse/app/federation_sender.py @@ -25,6 +25,7 @@ from synapse.config.logger import setup_logging from synapse.crypto import context_factory from synapse.federation import send_queue from synapse.http.site import SynapseSite +from synapse.metrics import RegistryProxy from synapse.metrics.resource import METRICS_PREFIX, MetricsResource from synapse.replication.slave.storage.deviceinbox import SlavedDeviceInboxStore from synapse.replication.slave.storage.devices import SlavedDeviceStore @@ -89,7 +90,7 @@ class FederationSenderServer(HomeServer): for res in listener_config["resources"]: for name in res["names"]: if name == "metrics": - resources[METRICS_PREFIX] = MetricsResource(self) + resources[METRICS_PREFIX] = MetricsResource(RegistryProxy) root_resource = create_resource_tree(resources, NoResource()) @@ -121,6 +122,13 @@ class FederationSenderServer(HomeServer): globals={"hs": self}, ) ) + elif listener["type"] == "metrics": + if not self.get_config().enable_metrics: + logger.warn(("Metrics listener configured, but " + "collect_metrics is not enabled!")) + else: + _base.listen_metrics(listener["bind_addresses"], + listener["port"]) else: logger.warn("Unrecognized listener type: %s", listener["type"]) diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py index 0f700ee786..5a164a7a95 100644 --- a/synapse/app/frontend_proxy.py +++ b/synapse/app/frontend_proxy.py @@ -29,6 +29,7 @@ from synapse.http.servlet import ( RestServlet, parse_json_object_from_request, ) from synapse.http.site import SynapseSite +from synapse.metrics import RegistryProxy from synapse.metrics.resource import METRICS_PREFIX, MetricsResource from synapse.replication.slave.storage._base import BaseSlavedStore from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore @@ -131,7 +132,7 @@ class FrontendProxyServer(HomeServer): for res in listener_config["resources"]: for name in res["names"]: if name == "metrics": - resources[METRICS_PREFIX] = MetricsResource(self) + resources[METRICS_PREFIX] = MetricsResource(RegistryProxy) elif name == "client": resource = JsonResource(self, canonical_json=False) KeyUploadServlet(self).register(resource) @@ -172,6 +173,13 @@ class FrontendProxyServer(HomeServer): globals={"hs": self}, ) ) + elif listener["type"] == "metrics": + if not self.get_config().enable_metrics: + logger.warn(("Metrics listener configured, but " + "collect_metrics is not enabled!")) + else: + _base.listen_metrics(listener["bind_addresses"], + listener["port"]) else: logger.warn("Unrecognized listener type: %s", listener["type"]) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 449bfacdb9..51fc3645d5 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -35,7 +35,7 @@ from synapse.http.additional_resource import AdditionalResource from synapse.http.server import RootRedirect from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy -from synapse.metrics.resource import METRICS_PREFIX +from synapse.metrics.resource import METRICS_PREFIX, MetricsResource from synapse.python_dependencies import CONDITIONAL_REQUIREMENTS, \ check_requirements from synapse.replication.http import ReplicationRestResource, REPLICATION_PREFIX @@ -61,8 +61,6 @@ from twisted.web.resource import EncodingResourceWrapper, NoResource from twisted.web.server import GzipEncoderFactory from twisted.web.static import File -from prometheus_client.twisted import MetricsResource - logger = logging.getLogger("synapse.app.homeserver") @@ -232,7 +230,7 @@ class SynapseHomeServer(HomeServer): resources[WEB_CLIENT_PREFIX] = build_resource_for_web_client(self) if name == "metrics" and self.get_config().enable_metrics: - resources[METRICS_PREFIX] = MetricsResource(RegistryProxy()) + resources[METRICS_PREFIX] = MetricsResource(RegistryProxy) if name == "replication": resources[REPLICATION_PREFIX] = ReplicationRestResource(self) @@ -265,6 +263,13 @@ class SynapseHomeServer(HomeServer): reactor.addSystemEventTrigger( "before", "shutdown", server_listener.stopListening, ) + elif listener["type"] == "metrics": + if not self.get_config().enable_metrics: + logger.warn(("Metrics listener configured, but " + "collect_metrics is not enabled!")) + else: + _base.listen_metrics(listener["bind_addresses"], + listener["port"]) else: logger.warn("Unrecognized listener type: %s", listener["type"]) diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py index 9c93195f0a..006bba80a8 100644 --- a/synapse/app/media_repository.py +++ b/synapse/app/media_repository.py @@ -27,6 +27,7 @@ from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging from synapse.crypto import context_factory from synapse.http.site import SynapseSite +from synapse.metrics import RegistryProxy from synapse.metrics.resource import METRICS_PREFIX, MetricsResource from synapse.replication.slave.storage._base import BaseSlavedStore from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore @@ -73,7 +74,7 @@ class MediaRepositoryServer(HomeServer): for res in listener_config["resources"]: for name in res["names"]: if name == "metrics": - resources[METRICS_PREFIX] = MetricsResource(self) + resources[METRICS_PREFIX] = MetricsResource(RegistryProxy) elif name == "media": media_repo = self.get_media_repository_resource() resources.update({ @@ -114,6 +115,13 @@ class MediaRepositoryServer(HomeServer): globals={"hs": self}, ) ) + elif listener["type"] == "metrics": + if not self.get_config().enable_metrics: + logger.warn(("Metrics listener configured, but " + "collect_metrics is not enabled!")) + else: + _base.listen_metrics(listener["bind_addresses"], + listener["port"]) else: logger.warn("Unrecognized listener type: %s", listener["type"]) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index 3912eae48c..64df47f9cc 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -23,6 +23,7 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging from synapse.http.site import SynapseSite +from synapse.metrics import RegistryProxy from synapse.metrics.resource import METRICS_PREFIX, MetricsResource from synapse.replication.slave.storage.account_data import SlavedAccountDataStore from synapse.replication.slave.storage.events import SlavedEventStore @@ -92,7 +93,7 @@ class PusherServer(HomeServer): for res in listener_config["resources"]: for name in res["names"]: if name == "metrics": - resources[METRICS_PREFIX] = MetricsResource(self) + resources[METRICS_PREFIX] = MetricsResource(RegistryProxy) root_resource = create_resource_tree(resources, NoResource()) @@ -124,6 +125,13 @@ class PusherServer(HomeServer): globals={"hs": self}, ) ) + elif listener["type"] == "metrics": + if not self.get_config().enable_metrics: + logger.warn(("Metrics listener configured, but " + "collect_metrics is not enabled!")) + else: + _base.listen_metrics(listener["bind_addresses"], + listener["port"]) else: logger.warn("Unrecognized listener type: %s", listener["type"]) diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index c6294a7a0c..6808d6d3e0 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -26,6 +26,7 @@ from synapse.config.logger import setup_logging from synapse.handlers.presence import PresenceHandler, get_interested_parties from synapse.http.server import JsonResource from synapse.http.site import SynapseSite +from synapse.metrics import RegistryProxy from synapse.metrics.resource import METRICS_PREFIX, MetricsResource from synapse.replication.slave.storage._base import BaseSlavedStore from synapse.replication.slave.storage.account_data import SlavedAccountDataStore @@ -257,7 +258,7 @@ class SynchrotronServer(HomeServer): for res in listener_config["resources"]: for name in res["names"]: if name == "metrics": - resources[METRICS_PREFIX] = MetricsResource(self) + resources[METRICS_PREFIX] = MetricsResource(RegistryProxy) elif name == "client": resource = JsonResource(self, canonical_json=False) sync.register_servlets(self, resource) @@ -301,6 +302,13 @@ class SynchrotronServer(HomeServer): globals={"hs": self}, ) ) + elif listener["type"] == "metrics": + if not self.get_config().enable_metrics: + logger.warn(("Metrics listener configured, but " + "collect_metrics is not enabled!")) + else: + _base.listen_metrics(listener["bind_addresses"], + listener["port"]) else: logger.warn("Unrecognized listener type: %s", listener["type"]) diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py index 53eb3474da..ada1c13cec 100644 --- a/synapse/app/user_dir.py +++ b/synapse/app/user_dir.py @@ -26,6 +26,7 @@ from synapse.config.logger import setup_logging from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.site import SynapseSite +from synapse.metrics import RegistryProxy from synapse.metrics.resource import METRICS_PREFIX, MetricsResource from synapse.replication.slave.storage._base import BaseSlavedStore from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore @@ -105,7 +106,7 @@ class UserDirectoryServer(HomeServer): for res in listener_config["resources"]: for name in res["names"]: if name == "metrics": - resources[METRICS_PREFIX] = MetricsResource(self) + resources[METRICS_PREFIX] = MetricsResource(RegistryProxy) elif name == "client": resource = JsonResource(self, canonical_json=False) user_directory.register_servlets(self, resource) @@ -146,6 +147,13 @@ class UserDirectoryServer(HomeServer): globals={"hs": self}, ) ) + elif listener["type"] == "metrics": + if not self.get_config().enable_metrics: + logger.warn(("Metrics listener configured, but " + "collect_metrics is not enabled!")) + else: + _base.listen_metrics(listener["bind_addresses"], + listener["port"]) else: logger.warn("Unrecognized listener type: %s", listener["type"]) diff --git a/synapse/config/server.py b/synapse/config/server.py index 8f0b6d1f28..968ecd9ea0 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -14,8 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging + from ._base import Config, ConfigError +logger = logging.Logger(__name__) + class ServerConfig(Config): @@ -138,6 +142,12 @@ class ServerConfig(Config): metrics_port = config.get("metrics_port") if metrics_port: + logger.warn( + ("The metrics_port configuration option is deprecated in Synapse 0.31 " + "in favour of a listener. Please see " + "http://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.rst" + " on how to configure the new listener.")) + self.listeners.append({ "port": metrics_port, "bind_addresses": [config.get("metrics_bind_host", "127.0.0.1")], diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index bfdbbc9a23..56c0032f91 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -39,7 +39,8 @@ HAVE_PROC_SELF_STAT = os.path.exists("/proc/self/stat") class RegistryProxy(object): - def collect(self): + @staticmethod + def collect(): for metric in REGISTRY.collect(): if not metric.name.startswith("__"): yield metric diff --git a/synapse/metrics/resource.py b/synapse/metrics/resource.py index 7996e6ab66..9789359077 100644 --- a/synapse/metrics/resource.py +++ b/synapse/metrics/resource.py @@ -13,4 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from prometheus_client.twisted import MetricsResource + METRICS_PREFIX = "/_synapse/metrics" + +__all__ = ["MetricsResource", "METRICS_PREFIX"] From c2c3092cce2057983fb99d096824bcb2204dbc82 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 31 May 2018 16:11:34 +0100 Subject: [PATCH 283/310] code_style.rst: formatting --- docs/code_style.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/code_style.rst b/docs/code_style.rst index 9c52cb3182..62800b5b3e 100644 --- a/docs/code_style.rst +++ b/docs/code_style.rst @@ -16,7 +16,7 @@ print("I am a fish %s" % "moo") - and this:: + and this:: print( "I am a fish %s" % From 4986b084f8e81e7697ea2022d306de03f8b0263e Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Fri, 1 Jun 2018 10:50:40 +0100 Subject: [PATCH 284/310] remove unnecessary INSERT --- synapse/storage/schema/delta/50/add_creation_ts_users_index.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/schema/delta/50/add_creation_ts_users_index.sql b/synapse/storage/schema/delta/50/add_creation_ts_users_index.sql index ba33acfd51..c93ae47532 100644 --- a/synapse/storage/schema/delta/50/add_creation_ts_users_index.sql +++ b/synapse/storage/schema/delta/50/add_creation_ts_users_index.sql @@ -15,6 +15,5 @@ -CREATE INDEX users_creation_ts ON users(creation_ts); INSERT into background_updates (update_name, progress_json) VALUES ('users_creation_ts', '{}'); From 4ef76f3ac4cec1e2faf84c5423eb1c305966b646 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=A4fer?= Date: Fri, 1 Jun 2018 12:18:35 +0200 Subject: [PATCH 285/310] Add private IPv6 addresses to preview blacklist #3312 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The added addresses are expected to be local or loopback addresses and shouldn't be spidered for previews. Signed-off-by: Felix Schäfer --- synapse/config/repository.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 25ea77738a..81ecf9778c 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -250,6 +250,9 @@ class ContentRepositoryConfig(Config): # - '192.168.0.0/16' # - '100.64.0.0/10' # - '169.254.0.0/16' + # - '::1/128' + # - 'fe80::/64' + # - 'fc00::/7' # # List of IP address CIDR ranges that the URL preview spider is allowed # to access even if they are specified in url_preview_ip_range_blacklist. From 857e6fd8b681ba3009943c2466f56fe4c3239933 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 1 Jun 2018 12:18:11 +0100 Subject: [PATCH 286/310] Ignore depth when updating read-receipts Order read receipts by stream ordering instead of depth --- synapse/storage/receipts.py | 67 ++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index 709c69a926..dd183cebce 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +from synapse.api.errors import NotFoundError from ._base import SQLBaseStore from .util.id_generators import StreamIdGenerator from synapse.util.caches.descriptors import cachedInlineCallbacks, cachedList, cached @@ -332,6 +332,41 @@ class ReceiptsStore(ReceiptsWorkerStore): def insert_linearized_receipt_txn(self, txn, room_id, receipt_type, user_id, event_id, data, stream_id): + res = self._simple_select_one_txn( + txn, + table="events", + retcols=["topological_ordering", "stream_ordering"], + keyvalues={"event_id": event_id}, + allow_none=True + ) + + if not res: + raise NotFoundError( + "Cannot set read receipt on unknown event %s" % ( + event_id, + ), + ) + + stream_ordering = int(res["stream_ordering"]) + + # We don't want to clobber receipts for more recent events, so we + # have to compare orderings of existing receipts + sql = ( + "SELECT stream_ordering, event_id FROM events" + " INNER JOIN receipts_linearized as r USING (event_id, room_id)" + " WHERE r.room_id = ? AND r.receipt_type = ? AND r.user_id = ?" + ) + txn.execute(sql, (room_id, receipt_type, user_id)) + + for so, eid in txn: + if int(so) >= stream_ordering: + logger.debug( + "Ignoring new receipt for %s in favour of existing " + "one for later event %s", + event_id, eid, + ) + return False + txn.call_after( self.get_receipts_for_room.invalidate, (room_id, receipt_type) ) @@ -355,34 +390,6 @@ class ReceiptsStore(ReceiptsWorkerStore): (user_id, room_id, receipt_type) ) - res = self._simple_select_one_txn( - txn, - table="events", - retcols=["topological_ordering", "stream_ordering"], - keyvalues={"event_id": event_id}, - allow_none=True - ) - - topological_ordering = int(res["topological_ordering"]) if res else None - stream_ordering = int(res["stream_ordering"]) if res else None - - # We don't want to clobber receipts for more recent events, so we - # have to compare orderings of existing receipts - sql = ( - "SELECT topological_ordering, stream_ordering, event_id FROM events" - " INNER JOIN receipts_linearized as r USING (event_id, room_id)" - " WHERE r.room_id = ? AND r.receipt_type = ? AND r.user_id = ?" - ) - - txn.execute(sql, (room_id, receipt_type, user_id)) - - if topological_ordering: - for to, so, _ in txn: - if int(to) > topological_ordering: - return False - elif int(to) == topological_ordering and int(so) >= stream_ordering: - return False - self._simple_delete_txn( txn, table="receipts_linearized", @@ -406,7 +413,7 @@ class ReceiptsStore(ReceiptsWorkerStore): } ) - if receipt_type == "m.read" and topological_ordering: + if receipt_type == "m.read": self._remove_old_push_actions_before_txn( txn, room_id=room_id, From 9f797a24a452a513628263b1b03172cee20a9856 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 1 Jun 2018 14:01:43 +0100 Subject: [PATCH 287/310] Handle RRs which arrive before their events --- synapse/storage/receipts.py | 42 ++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index dd183cebce..c93c228f6e 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from synapse.api.errors import NotFoundError + from ._base import SQLBaseStore from .util.id_generators import StreamIdGenerator from synapse.util.caches.descriptors import cachedInlineCallbacks, cachedList, cached @@ -340,32 +340,26 @@ class ReceiptsStore(ReceiptsWorkerStore): allow_none=True ) - if not res: - raise NotFoundError( - "Cannot set read receipt on unknown event %s" % ( - event_id, - ), - ) - - stream_ordering = int(res["stream_ordering"]) + stream_ordering = int(res["stream_ordering"]) if res else None # We don't want to clobber receipts for more recent events, so we # have to compare orderings of existing receipts - sql = ( - "SELECT stream_ordering, event_id FROM events" - " INNER JOIN receipts_linearized as r USING (event_id, room_id)" - " WHERE r.room_id = ? AND r.receipt_type = ? AND r.user_id = ?" - ) - txn.execute(sql, (room_id, receipt_type, user_id)) + if stream_ordering is not None: + sql = ( + "SELECT stream_ordering, event_id FROM events" + " INNER JOIN receipts_linearized as r USING (event_id, room_id)" + " WHERE r.room_id = ? AND r.receipt_type = ? AND r.user_id = ?" + ) + txn.execute(sql, (room_id, receipt_type, user_id)) - for so, eid in txn: - if int(so) >= stream_ordering: - logger.debug( - "Ignoring new receipt for %s in favour of existing " - "one for later event %s", - event_id, eid, - ) - return False + for so, eid in txn: + if int(so) >= stream_ordering: + logger.debug( + "Ignoring new receipt for %s in favour of existing " + "one for later event %s", + event_id, eid, + ) + return False txn.call_after( self.get_receipts_for_room.invalidate, (room_id, receipt_type) @@ -413,7 +407,7 @@ class ReceiptsStore(ReceiptsWorkerStore): } ) - if receipt_type == "m.read": + if receipt_type == "m.read" and stream_ordering is not None: self._remove_old_push_actions_before_txn( txn, room_id=room_id, From 7d9d75e4e80eef0f3569c57d38842c743ebd03f9 Mon Sep 17 00:00:00 2001 From: Ivan Shapovalov Date: Sun, 3 Jun 2018 14:14:47 +0300 Subject: [PATCH 288/310] federation/send_queue.py: fix usage of LaterGauge Fixes a startup crash due to commit df9f72d9e5fe264b86005208e0f096156eb03e4b "replacing portions". --- synapse/federation/send_queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py index 3dcc629d44..9f1142b5a9 100644 --- a/synapse/federation/send_queue.py +++ b/synapse/federation/send_queue.py @@ -75,7 +75,7 @@ class FederationRemoteSendQueue(object): # changes. ARGH. def register(name, queue): LaterGauge("synapse_federation_send_queue_%s_size" % (queue_name,), - "", lambda: len(queue)) + "", [], lambda: len(queue)) for queue_name in [ "presence_map", "presence_changed", "keyed_edu", "keyed_edu_changed", From 5dbf3054446a2d772eafc1b8a421bbb2edcae425 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 4 Jun 2018 16:06:06 +1000 Subject: [PATCH 289/310] Put python's logs into Trial when running unit tests (#3319) --- tests/unittest.py | 42 ++++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/tests/unittest.py b/tests/unittest.py index 7b478c4294..184fe880f3 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -12,23 +12,37 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import twisted -from twisted.trial import unittest import logging -# logging doesn't have a "don't log anything at all EVARRRR setting, -# but since the highest value is 50, 1000000 should do ;) -NEVER = 1000000 +import twisted +import twisted.logger +from twisted.trial import unittest -handler = logging.StreamHandler() -handler.setFormatter(logging.Formatter( - "%(levelname)s:%(name)s:%(message)s [%(pathname)s:%(lineno)d]" -)) -logging.getLogger().addHandler(handler) -logging.getLogger().setLevel(NEVER) -logging.getLogger("synapse.storage.SQL").setLevel(NEVER) -logging.getLogger("synapse.storage.txn").setLevel(NEVER) +from synapse.util.logcontext import LoggingContextFilter + +# Set up putting Synapse's logs into Trial's. +rootLogger = logging.getLogger() + +log_format = ( + "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s" +) + + +class ToTwistedHandler(logging.Handler): + tx_log = twisted.logger.Logger() + + def emit(self, record): + log_entry = self.format(record) + log_level = record.levelname.lower().replace('warning', 'warn') + self.tx_log.emit(twisted.logger.LogLevel.levelWithName(log_level), log_entry) + + +handler = ToTwistedHandler() +formatter = logging.Formatter(log_format) +handler.setFormatter(formatter) +handler.addFilter(LoggingContextFilter(request="")) +rootLogger.addHandler(handler) def around(target): @@ -61,7 +75,7 @@ class TestCase(unittest.TestCase): method = getattr(self, methodName) - level = getattr(method, "loglevel", getattr(self, "loglevel", NEVER)) + level = getattr(method, "loglevel", getattr(self, "loglevel", logging.ERROR)) @around(self) def setUp(orig): From 694968fa81aab4eac81309b1e16f6063103dd57f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 4 Jun 2018 15:59:14 +0100 Subject: [PATCH 290/310] Hopefully, fix LaterGuage error handling --- synapse/metrics/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index 56c0032f91..429e79c472 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -60,10 +60,13 @@ class LaterGauge(object): try: calls = self.caller() - except Exception as e: - print(e) - logger.err() + except Exception: + logger.exception( + "Exception running callback for LaterGuage(%s)", + self.name, + ) yield g + return if isinstance(calls, dict): for k, v in calls.items(): From 244ab974e7e17f3c1688a53bd3258271c644a82e Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 4 Jun 2018 16:09:58 +0100 Subject: [PATCH 291/310] bump version and changelog --- CHANGES.rst | 46 +++++++++++++++++++++++++++++++++++++++++++++ synapse/__init__.py | 2 +- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0569b581db..531d9ed151 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,49 @@ +Changes in synapse v0.31.0-rc1 (2018-06-04) +========================================== + +Most notable change is to switch to python prometheus library to improve system stats reporting + +Features: + +* Switch to the Python Prometheus library (PR #3256, #3274) +* Let users leave the server notice room after joining (PR #3287) + + +Changes: + +* daily user type phone home stats (PR #3264) +* Use iter* methods for _filter_events_for_server (PR #3267) +* Docs on consent bits (PR #3268) +* Remove users from user directory on deactivate (PR #3277) +* Avoid sending consent notice to guest users (PR #3288) +* disable CPUMetrics if no /proc/self/stat (PR #3299) +* Add local and loopback IPv6 addresses to url_preview_ip_range_blacklist (PR #3312) Thanks to @thegcat! +* Consistently use six's iteritems and wrap lazy keys/values in list() if they're not meant to be lazy (PR #3307) +* Add private IPv6 addresses to example config for url preview blacklist (PR #3317) Thanks to @thegcat! +* Reduce stuck read-receipts: ignore depth when updating (PR #3318) +* Put python's logs into Trial when running unit tests (PR #3319) + +Changes, python 3 migration: + +* Replace some more comparisons with six (PR #3243) Thanks to @NotAFile! +* replace some iteritems with six (PR #3244) Thanks to @NotAFile! +* Add batch_iter to utils (PR #3245) Thanks to @NotAFile! +* use repr, not str (PR #3246) Thanks to @NotAFile! +* Misc Python3 fixes (PR #3247) Thanks to @NotAFile! +* Py3 storage/_base.py (PR #3278) Thanks to @NotAFile! +* more six iteritems (PR #3279) Thanks to @NotAFile! +* More Misc. py3 fixes (PR #3280) Thanks to @NotAFile! +* remaining isintance fixes (PR #3281) Thanks to @NotAFile! +* py3-ize state.py (PR #3283) Thanks to @NotAFile! +* extend tox testing for py3 to avoid regressions (PR #3302) Thanks to @krombel! +* use memoryview in py3 (PR #3303) Thanks to @NotAFile! + +Bugs: + +* Fix federation backfill bugs (PR #3261) +* federation: fix LaterGauge usage (PR #3328) Thanks to @intelfx! + + Changes in synapse v0.30.0 (2018-05-24) ========================================== diff --git a/synapse/__init__.py b/synapse/__init__.py index 5bada5e290..a1ad7830c9 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.30.0" +__version__ = "0.31.0-rc1" From b7e7fd2d0edd4d46aaf9d6afc8df14cf3de911f9 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 4 Jun 2018 16:23:05 +0100 Subject: [PATCH 292/310] Fix replication metrics fix bug introduced in #3256 --- synapse/replication/tcp/protocol.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py index a6280aae70..c870475cd1 100644 --- a/synapse/replication/tcp/protocol.py +++ b/synapse/replication/tcp/protocol.py @@ -622,7 +622,7 @@ tcp_inbound_commands = LaterGauge( lambda: { (k[0], p.name, p.conn_id): count for p in connected_connections - for k, count in iteritems(p.inbound_commands_counter.counts) + for k, count in iteritems(p.inbound_commands_counter) }) tcp_outbound_commands = LaterGauge( @@ -630,7 +630,7 @@ tcp_outbound_commands = LaterGauge( lambda: { (k[0], p.name, p.conn_id): count for p in connected_connections - for k, count in iteritems(p.outbound_commands_counter.counts) + for k, count in iteritems(p.outbound_commands_counter) }) # number of updates received for each RDATA stream From b3b16490f7eafea7fe268f4fc1a86593e49e10a1 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 6 Jun 2018 07:08:36 +0100 Subject: [PATCH 293/310] Add note to changelog on prometheus metrics --- CHANGES.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 531d9ed151..be9429f4e2 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,7 +1,10 @@ Changes in synapse v0.31.0-rc1 (2018-06-04) ========================================== -Most notable change is to switch to python prometheus library to improve system stats reporting +Most notable change is to switch to python prometheus library to improve system +stats reporting. WARNING this changes a number of prometheus metrics in a +backwards-incompatible manner. For more details, see +`docs/metrics-howto.rst `_. Features: From 23c785992f685f0b5dedfc863bdd55e990751561 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 6 Jun 2018 15:52:37 +1000 Subject: [PATCH 294/310] Fix metric documentation tables (#3341) --- docs/metrics-howto.rst | 50 +++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/docs/metrics-howto.rst b/docs/metrics-howto.rst index 25e06bca58..5bbb5a4f3a 100644 --- a/docs/metrics-howto.rst +++ b/docs/metrics-howto.rst @@ -63,30 +63,40 @@ The duplicated metrics deprecated in Synapse 0.27.0 have been removed. All time duration-based metrics have been changed to be seconds. This affects: -================================ -msec -> sec metrics -================================ -python_gc_time -python_twisted_reactor_tick_time -synapse_storage_query_time -synapse_storage_schedule_time -synapse_storage_transaction_time -================================ ++----------------------------------+ +| msec -> sec metrics | ++==================================+ +| python_gc_time | ++----------------------------------+ +| python_twisted_reactor_tick_time | ++----------------------------------+ +| synapse_storage_query_time | ++----------------------------------+ +| synapse_storage_schedule_time | ++----------------------------------+ +| synapse_storage_transaction_time | ++----------------------------------+ Several metrics have been changed to be histograms, which sort entries into buckets and allow better analysis. The following metrics are now histograms: -========================================= -Altered metrics -========================================= -python_gc_time -python_twisted_reactor_pending_calls -python_twisted_reactor_tick_time -synapse_http_server_response_time_seconds -synapse_storage_query_time -synapse_storage_schedule_time -synapse_storage_transaction_time -========================================= ++-------------------------------------------+ +| Altered metrics | ++===========================================+ +| python_gc_time | ++-------------------------------------------+ +| python_twisted_reactor_pending_calls | ++-------------------------------------------+ +| python_twisted_reactor_tick_time | ++-------------------------------------------+ +| synapse_http_server_response_time_seconds | ++-------------------------------------------+ +| synapse_storage_query_time | ++-------------------------------------------+ +| synapse_storage_schedule_time | ++-------------------------------------------+ +| synapse_storage_transaction_time | ++-------------------------------------------+ Block and response metrics renamed for 0.27.0 From 61134debdc86d415eecefc958981f740837ee619 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 6 Jun 2018 11:26:21 +0100 Subject: [PATCH 295/310] bump version and changelog --- CHANGES.rst | 13 +++++++++++-- synapse/__init__.py | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index be9429f4e2..ee104a7e9c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,11 +1,20 @@ -Changes in synapse v0.31.0-rc1 (2018-06-04) -========================================== +Changes in synapse v0.31.0 (2018-06-06) +======================================= Most notable change is to switch to python prometheus library to improve system stats reporting. WARNING this changes a number of prometheus metrics in a backwards-incompatible manner. For more details, see `docs/metrics-howto.rst `_. +Bug Fixes: + + * Fix metric documentation tables (PR #3341) + * Fix LaterGuage error handling (694968fa81aab4eac81309b1e16f6063103dd57f) + * Fix replication metrics (b7e7fd2d0edd4d46aaf9d6afc8df14cf3de911f9) + +Changes in synapse v0.31.0-rc1 (2018-06-04) +========================================== + Features: * Switch to the Python Prometheus library (PR #3256, #3274) diff --git a/synapse/__init__.py b/synapse/__init__.py index a1ad7830c9..ca113db434 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.31.0-rc1" +__version__ = "0.31.0" From 176f1206d1431f4dfbdb306cc3e673d7f7f44c4a Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 6 Jun 2018 11:28:30 +0100 Subject: [PATCH 296/310] Update CHANGES.rst --- CHANGES.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index ee104a7e9c..5c49e12630 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,10 +8,10 @@ backwards-incompatible manner. For more details, see Bug Fixes: - * Fix metric documentation tables (PR #3341) - * Fix LaterGuage error handling (694968fa81aab4eac81309b1e16f6063103dd57f) - * Fix replication metrics (b7e7fd2d0edd4d46aaf9d6afc8df14cf3de911f9) - +* Fix metric documentation tables (PR #3341) +* Fix LaterGuage error handling (694968fa81aab4eac81309b1e16f6063103dd57f) +* Fix replication metrics (b7e7fd2d0edd4d46aaf9d6afc8df14cf3de911f9) + Changes in synapse v0.31.0-rc1 (2018-06-04) ========================================== From 3f589f90974586ffccacc2641547452903955b60 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 6 Jun 2018 11:39:42 +0100 Subject: [PATCH 297/310] 7 char sha in changelog --- CHANGES.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 5c49e12630..f2b7f04097 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,7 +1,7 @@ Changes in synapse v0.31.0 (2018-06-06) ======================================= -Most notable change is to switch to python prometheus library to improve system +Most notable change from v0.30.0 is to switch to python prometheus library to improve system stats reporting. WARNING this changes a number of prometheus metrics in a backwards-incompatible manner. For more details, see `docs/metrics-howto.rst `_. @@ -9,8 +9,8 @@ backwards-incompatible manner. For more details, see Bug Fixes: * Fix metric documentation tables (PR #3341) -* Fix LaterGuage error handling (694968fa81aab4eac81309b1e16f6063103dd57f) -* Fix replication metrics (b7e7fd2d0edd4d46aaf9d6afc8df14cf3de911f9) +* Fix LaterGuage error handling (694968f) +* Fix replication metrics (b7e7fd2) Changes in synapse v0.31.0-rc1 (2018-06-04) ========================================== From 8b98acca05fcf6b4a64ce27a27df398b922039fa Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 7 Jun 2018 23:54:16 +0300 Subject: [PATCH 298/310] fix various changelog bugs and typos --- CHANGES.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index f2b7f04097..3a2a30873f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,15 +1,15 @@ Changes in synapse v0.31.0 (2018-06-06) ======================================= -Most notable change from v0.30.0 is to switch to python prometheus library to improve system -stats reporting. WARNING this changes a number of prometheus metrics in a +Most notable change from v0.30.0 is to switch to the python prometheus library to improve system +stats reporting. WARNING: this changes a number of prometheus metrics in a backwards-incompatible manner. For more details, see `docs/metrics-howto.rst `_. Bug Fixes: * Fix metric documentation tables (PR #3341) -* Fix LaterGuage error handling (694968f) +* Fix LaterGauge error handling (694968f) * Fix replication metrics (b7e7fd2) Changes in synapse v0.31.0-rc1 (2018-06-04) @@ -29,7 +29,6 @@ Changes: * Remove users from user directory on deactivate (PR #3277) * Avoid sending consent notice to guest users (PR #3288) * disable CPUMetrics if no /proc/self/stat (PR #3299) -* Add local and loopback IPv6 addresses to url_preview_ip_range_blacklist (PR #3312) Thanks to @thegcat! * Consistently use six's iteritems and wrap lazy keys/values in list() if they're not meant to be lazy (PR #3307) * Add private IPv6 addresses to example config for url preview blacklist (PR #3317) Thanks to @thegcat! * Reduce stuck read-receipts: ignore depth when updating (PR #3318) From c6b1441c52e06b6b8c8715dd3e4d4340aac3a216 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 8 Jun 2018 11:34:46 +0100 Subject: [PATCH 299/310] Fix event filtering in get_missing_events handler --- synapse/handlers/federation.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index fcf94befb7..495ac4c648 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1794,6 +1794,10 @@ class FederationHandler(BaseHandler): min_depth=min_depth, ) + missing_events = yield self._filter_events_for_server( + origin, room_id, missing_events, + ) + defer.returnValue(missing_events) @defer.inlineCallbacks From 0eb4722932b4d2d92cd8cca97e1b36bd3b20d38d Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Fri, 8 Jun 2018 15:21:46 +0100 Subject: [PATCH 300/310] changelog a bump version --- CHANGES.rst | 10 ++++++++++ synapse/__init__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 3a2a30873f..ff137f481a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,13 @@ +Changes in synapse v0.31.1 (2018-06-08) +======================================= + +This release fixes a security bug where event visibility rules were not +applied correctly to ``get_missing_events``. + +Bug Fixes: + + * Fix event filtering in get_missing_events handler (PR #3371) + Changes in synapse v0.31.0 (2018-06-06) ======================================= diff --git a/synapse/__init__.py b/synapse/__init__.py index ca113db434..78fc63aa49 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.31.0" +__version__ = "0.31.1" From 82e751c43fceb9fc20127e766cab59c9c376d4cb Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Fri, 8 Jun 2018 15:22:34 +0100 Subject: [PATCH 301/310] Update CHANGES.rst --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index ff137f481a..397509e9aa 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,7 +6,7 @@ applied correctly to ``get_missing_events``. Bug Fixes: - * Fix event filtering in get_missing_events handler (PR #3371) +* Fix event filtering in get_missing_events handler (PR #3371) Changes in synapse v0.31.0 (2018-06-06) ======================================= From aefcc0f5e5e3b4ce6020dd895045ddf5f83ba5d9 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Fri, 8 Jun 2018 15:32:54 +0100 Subject: [PATCH 302/310] tweak changelog --- CHANGES.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 397509e9aa..4047f50aa5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,8 +1,10 @@ Changes in synapse v0.31.1 (2018-06-08) ======================================= -This release fixes a security bug where event visibility rules were not -applied correctly to ``get_missing_events``. +v0.31.1 fixes a security bug in the ``get_missing_events`` federation API +where event visibility rules were not applied correctly. + +We are not aware of it being actively exploited but please upgrade asap. Bug Fixes: From 557b686eacca0b59820489daf6a5ea929cda1636 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 14 Jun 2018 11:26:27 +0100 Subject: [PATCH 303/310] Refactor get_send_level to take a power_levels event it makes it easier for me to reason about --- synapse/api/auth.py | 2 +- synapse/event_auth.py | 45 ++++++++++++++++++++++++++++++++++++------- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 06fa38366d..66639b0089 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -655,7 +655,7 @@ class Auth(object): auth_events[(EventTypes.PowerLevels, "")] = power_level_event send_level = event_auth.get_send_level( - EventTypes.Aliases, "", auth_events + EventTypes.Aliases, "", power_level_event, ) user_level = event_auth.get_user_power_level(user_id, auth_events) diff --git a/synapse/event_auth.py b/synapse/event_auth.py index eaf9cecde6..138175a554 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -355,9 +355,24 @@ def _check_joined_room(member, user_id, room_id): )) -def get_send_level(etype, state_key, auth_events): - key = (EventTypes.PowerLevels, "", ) - send_level_event = auth_events.get(key) +def get_send_level(etype, state_key, power_levels_event): + """Get the power level required to send an event of a given type + + The federation spec [1] refers to this as "Required Power Level". + + https://matrix.org/docs/spec/server_server/unstable.html#definitions + + Args: + etype (str): type of event + state_key (str|None): state_key of state event, or None if it is not + a state event. + power_levels_event (synapse.events.EventBase|None): power levels event + in force at this point in the room + Returns: + int: power level required to send this event. + """ + + send_level_event = power_levels_event # todo: rename refs below send_level = None if send_level_event: send_level = send_level_event.content.get("events", {}).get( @@ -382,8 +397,10 @@ def get_send_level(etype, state_key, auth_events): def _can_send_event(event, auth_events): + power_levels_event = _get_power_level_event(auth_events) + send_level = get_send_level( - event.type, event.get("state_key", None), auth_events + event.type, event.get("state_key"), power_levels_event, ) user_level = get_user_power_level(event.user_id, auth_events) @@ -524,13 +541,22 @@ def _check_power_levels(event, auth_events): def _get_power_level_event(auth_events): - key = (EventTypes.PowerLevels, "", ) - return auth_events.get(key) + return auth_events.get((EventTypes.PowerLevels, "")) def get_user_power_level(user_id, auth_events): - power_level_event = _get_power_level_event(auth_events) + """Get a user's power level + Args: + user_id (str): user's id to look up in power_levels + auth_events (dict[(str, str), synapse.events.EventBase]): + state in force at this point in the room (or rather, a subset of + it including at least the create event and power levels event. + + Returns: + int: the user's power level in this room. + """ + power_level_event = _get_power_level_event(auth_events) if power_level_event: level = power_level_event.content.get("users", {}).get(user_id) if not level: @@ -541,6 +567,11 @@ def get_user_power_level(user_id, auth_events): else: return int(level) else: + # if there is no power levels event, the creator gets 100 and everyone + # else gets 0. + + # some things which call this don't pass the create event: hack around + # that. key = (EventTypes.Create, "", ) create_event = auth_events.get(key) if (create_event is not None and From 52423607bddf6c1bae5dbc89fb2c94496ad70fde Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 14 Jun 2018 12:26:17 +0100 Subject: [PATCH 304/310] Clarify interface for event_auth stop pretending that it returns a boolean, which just almost gave me a heart attack. --- synapse/event_auth.py | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/synapse/event_auth.py b/synapse/event_auth.py index 138175a554..36a48870a0 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -34,9 +34,11 @@ def check(event, auth_events, do_sig_check=True, do_size_check=True): event: the event being checked. auth_events (dict: event-key -> event): the existing room state. + Raises: + AuthError if the checks fail Returns: - True if the auth checks pass. + if the auth checks pass. """ if do_size_check: _check_size_limits(event) @@ -71,7 +73,7 @@ def check(event, auth_events, do_sig_check=True, do_size_check=True): # Oh, we don't know what the state of the room was, so we # are trusting that this is allowed (at least for now) logger.warn("Trusting event: %s", event.event_id) - return True + return if event.type == EventTypes.Create: room_id_domain = get_domain_from_id(event.room_id) @@ -81,7 +83,8 @@ def check(event, auth_events, do_sig_check=True, do_size_check=True): "Creation event's room_id domain does not match sender's" ) # FIXME - return True + logger.debug("Allowing! %s", event) + return creation_event = auth_events.get((EventTypes.Create, ""), None) @@ -118,7 +121,8 @@ def check(event, auth_events, do_sig_check=True, do_size_check=True): 403, "Alias event's state_key does not match sender's domain" ) - return True + logger.debug("Allowing! %s", event) + return if logger.isEnabledFor(logging.DEBUG): logger.debug( @@ -127,14 +131,9 @@ def check(event, auth_events, do_sig_check=True, do_size_check=True): ) if event.type == EventTypes.Member: - allowed = _is_membership_change_allowed( - event, auth_events - ) - if allowed: - logger.debug("Allowing! %s", event) - else: - logger.debug("Denying! %s", event) - return allowed + _is_membership_change_allowed(event, auth_events) + logger.debug("Allowing! %s", event) + return _check_event_sender_in_room(event, auth_events) @@ -153,7 +152,8 @@ def check(event, auth_events, do_sig_check=True, do_size_check=True): ) ) else: - return True + logger.debug("Allowing! %s", event) + return _can_send_event(event, auth_events) @@ -200,7 +200,7 @@ def _is_membership_change_allowed(event, auth_events): create = auth_events.get(key) if create and event.prev_events[0][0] == create.event_id: if create.content["creator"] == event.state_key: - return True + return target_user_id = event.state_key @@ -265,13 +265,13 @@ def _is_membership_change_allowed(event, auth_events): raise AuthError( 403, "%s is banned from the room" % (target_user_id,) ) - return True + return if Membership.JOIN != membership: if (caller_invited and Membership.LEAVE == membership and target_user_id == event.user_id): - return True + return if not caller_in_room: # caller isn't joined raise AuthError( @@ -334,8 +334,6 @@ def _is_membership_change_allowed(event, auth_events): else: raise AuthError(500, "Unknown membership %s" % membership) - return True - def _check_event_sender_in_room(event, auth_events): key = (EventTypes.Member, event.user_id, ) From 5c9afd6f80cf04367fe9b02c396af9f85e02a611 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 14 Jun 2018 12:28:36 +0100 Subject: [PATCH 305/310] Make default state_default 50 Make it so that, before there is a power-levels event in the room, you need a power level of at least 50 to send state. Partially addresses https://github.com/matrix-org/matrix-doc/issues/1192 --- synapse/event_auth.py | 34 ++++----- tests/test_event_auth.py | 153 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 20 deletions(-) create mode 100644 tests/test_event_auth.py diff --git a/synapse/event_auth.py b/synapse/event_auth.py index 36a48870a0..f512d88145 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -370,28 +370,22 @@ def get_send_level(etype, state_key, power_levels_event): int: power level required to send this event. """ - send_level_event = power_levels_event # todo: rename refs below - send_level = None - if send_level_event: - send_level = send_level_event.content.get("events", {}).get( - etype - ) - if send_level is None: - if state_key is not None: - send_level = send_level_event.content.get( - "state_default", 50 - ) - else: - send_level = send_level_event.content.get( - "events_default", 0 - ) - - if send_level: - send_level = int(send_level) + if power_levels_event: + power_levels_content = power_levels_event.content else: - send_level = 0 + power_levels_content = {} - return send_level + # see if we have a custom level for this event type + send_level = power_levels_content.get("events", {}).get(etype) + + # otherwise, fall back to the state_default/events_default. + if send_level is None: + if state_key is not None: + send_level = power_levels_content.get("state_default", 50) + else: + send_level = power_levels_content.get("events_default", 0) + + return int(send_level) def _can_send_event(event, auth_events): diff --git a/tests/test_event_auth.py b/tests/test_event_auth.py new file mode 100644 index 0000000000..3d756b0d85 --- /dev/null +++ b/tests/test_event_auth.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse import event_auth +from synapse.api.errors import AuthError +from synapse.events import FrozenEvent +import unittest + + +class EventAuthTestCase(unittest.TestCase): + @unittest.DEBUG + def test_random_users_cannot_send_state_before_first_pl(self): + """ + Check that, before the first PL lands, the creator is the only user + that can send a state event. + """ + creator = "@creator:example.com" + joiner = "@joiner:example.com" + auth_events = { + ("m.room.create", ""): _create_event(creator), + ("m.room.member", creator): _join_event(creator), + ("m.room.member", joiner): _join_event(joiner), + } + + # creator should be able to send state + event_auth.check( + _random_state_event(creator), auth_events, + do_sig_check=False, + ) + + # joiner should not be able to send state + self.assertRaises( + AuthError, + event_auth.check, + _random_state_event(joiner), + auth_events, + do_sig_check=False, + ), + + @unittest.DEBUG + def test_state_default_level(self): + """ + Check that users above the state_default level can send state and + those below cannot + """ + creator = "@creator:example.com" + pleb = "@joiner:example.com" + king = "@joiner2:example.com" + + auth_events = { + ("m.room.create", ""): _create_event(creator), + ("m.room.member", creator): _join_event(creator), + ("m.room.power_levels", ""): _power_levels_event(creator, { + "state_default": "30", + "users": { + pleb: "29", + king: "30", + }, + }), + ("m.room.member", pleb): _join_event(pleb), + ("m.room.member", king): _join_event(king), + } + + # pleb should not be able to send state + self.assertRaises( + AuthError, + event_auth.check, + _random_state_event(pleb), + auth_events, + do_sig_check=False, + ), + + # king should be able to send state + event_auth.check( + _random_state_event(king), auth_events, + do_sig_check=False, + ) + + +# helpers for making events + +TEST_ROOM_ID = "!test:room" + + +def _create_event(user_id): + return FrozenEvent({ + "room_id": TEST_ROOM_ID, + "event_id": _get_event_id(), + "type": "m.room.create", + "sender": user_id, + "content": { + "creator": user_id, + }, + }) + + +def _join_event(user_id): + return FrozenEvent({ + "room_id": TEST_ROOM_ID, + "event_id": _get_event_id(), + "type": "m.room.member", + "sender": user_id, + "state_key": user_id, + "content": { + "membership": "join", + }, + }) + + +def _power_levels_event(sender, content): + return FrozenEvent({ + "room_id": TEST_ROOM_ID, + "event_id": _get_event_id(), + "type": "m.room.power_levels", + "sender": sender, + "state_key": "", + "content": content, + }) + + +def _random_state_event(sender): + return FrozenEvent({ + "room_id": TEST_ROOM_ID, + "event_id": _get_event_id(), + "type": "test.state", + "sender": sender, + "state_key": "", + "content": { + "membership": "join", + }, + }) + + +event_count = 0 + + +def _get_event_id(): + global event_count + c = event_count + event_count += 1 + return "!%i:example.com" % (c, ) From a502cfec0054c91d314887771ec7d3aaa8491701 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 14 Jun 2018 14:20:53 +0100 Subject: [PATCH 306/310] remove spurious debug --- tests/test_event_auth.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_event_auth.py b/tests/test_event_auth.py index 3d756b0d85..d08e19c53a 100644 --- a/tests/test_event_auth.py +++ b/tests/test_event_auth.py @@ -20,7 +20,6 @@ import unittest class EventAuthTestCase(unittest.TestCase): - @unittest.DEBUG def test_random_users_cannot_send_state_before_first_pl(self): """ Check that, before the first PL lands, the creator is the only user @@ -49,7 +48,6 @@ class EventAuthTestCase(unittest.TestCase): do_sig_check=False, ), - @unittest.DEBUG def test_state_default_level(self): """ Check that users above the state_default level can send state and From 1e77ac66e38eca4b4da0c3f95efc3300504292e4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 14 Jun 2018 14:21:29 +0100 Subject: [PATCH 307/310] Fix broken unit test We need power levels for this test to do what it is supposed to do. --- tests/test_state.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/test_state.py b/tests/test_state.py index a5c5e55951..71c412faf4 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -606,6 +606,14 @@ class StateTestCase(unittest.TestCase): } ) + power_levels = create_event( + type=EventTypes.PowerLevels, state_key="", + content={"users": { + "@foo:bar": "100", + "@user_id:example.com": "100", + }} + ) + creation = create_event( type=EventTypes.Create, state_key="", content={"creator": "@foo:bar"} @@ -613,12 +621,14 @@ class StateTestCase(unittest.TestCase): old_state_1 = [ creation, + power_levels, member_event, create_event(type="test1", state_key="1", depth=1), ] old_state_2 = [ creation, + power_levels, member_event, create_event(type="test1", state_key="1", depth=2), ] @@ -633,7 +643,7 @@ class StateTestCase(unittest.TestCase): ) self.assertEqual( - old_state_2[2].event_id, context.current_state_ids[("test1", "1")] + old_state_2[3].event_id, context.current_state_ids[("test1", "1")] ) # Reverse the depth to make sure we are actually using the depths @@ -641,12 +651,14 @@ class StateTestCase(unittest.TestCase): old_state_1 = [ creation, + power_levels, member_event, create_event(type="test1", state_key="1", depth=2), ] old_state_2 = [ creation, + power_levels, member_event, create_event(type="test1", state_key="1", depth=1), ] @@ -659,7 +671,7 @@ class StateTestCase(unittest.TestCase): ) self.assertEqual( - old_state_1[2].event_id, context.current_state_ids[("test1", "1")] + old_state_1[3].event_id, context.current_state_ids[("test1", "1")] ) def _get_context(self, event, prev_event_id_1, old_state_1, prev_event_id_2, From ba438a3ac1e288e5dc21d741a8c16c9c5b45e27c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 14 Jun 2018 16:21:13 +0100 Subject: [PATCH 308/310] changelog for 0.31.2 --- CHANGES.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 4047f50aa5..08d11301e7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +Changes in synapse v0.31.2 (2018-06-14) +======================================= + +SECURITY UPDATE: Prevent unauthorised users from setting state events in a room +when there is no ``m.room.power_levels`` event in force in the room. (PR #3397) + Changes in synapse v0.31.1 (2018-06-08) ======================================= From 7e1c6164528ee0081ceab754c27b15c656337963 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 14 Jun 2018 16:24:32 +0100 Subject: [PATCH 309/310] v0.31.2 --- synapse/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 78fc63aa49..faa183a99e 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,4 +17,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.31.1" +__version__ = "0.31.2" From 667c6546bdbb50c1b5a88effe02ae12322d0d95b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 14 Jun 2018 16:27:41 +0100 Subject: [PATCH 310/310] link to spec proposal from changelog --- CHANGES.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 08d11301e7..70fc5af4c1 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,9 @@ Changes in synapse v0.31.2 (2018-06-14) SECURITY UPDATE: Prevent unauthorised users from setting state events in a room when there is no ``m.room.power_levels`` event in force in the room. (PR #3397) +Discussion around the Matrix Spec change proposal for this change can be +followed at https://github.com/matrix-org/matrix-doc/issues/1304. + Changes in synapse v0.31.1 (2018-06-08) =======================================