diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index d77d7792f0..14092a307a 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -91,10 +91,19 @@ jobs: rm -rf /tmp/.buildx-cache mv /tmp/.buildx-cache-new /tmp/.buildx-cache + - name: Artifact name + id: artifact-name + # We can't have colons in the upload name of the artifact, so we convert + # e.g. `debian:sid` to `sid`. + env: + DISTRO: ${{ matrix.distro }} + run: | + echo "ARTIFACT_NAME=${DISTRO#*:}" >> "$GITHUB_OUTPUT" + - name: Upload debs as artifacts uses: actions/upload-artifact@v4 with: - name: debs + name: debs-${{ steps.artifact-name.outputs.ARTIFACT_NAME }} path: debs/* build-wheels: @@ -196,7 +205,12 @@ jobs: - name: Download all workflow run artifacts uses: actions/download-artifact@v4 - name: Build a tarball for the debs - run: tar -cvJf debs.tar.xz debs + # We need to merge all the debs uploads into one folder, then compress + # that. + run: | + mkdir debs + mv debs*/* debs/ + tar -cvJf debs.tar.xz debs - name: Attach to release uses: softprops/action-gh-release@a929a66f232c1b11af63782948aa2210f981808a # PR#109 env: @@ -204,7 +218,7 @@ jobs: with: files: | Sdist/* - Wheel/* + Wheel*/* debs.tar.xz # if it's not already published, keep the release as a draft. draft: true diff --git a/CHANGES.md b/CHANGES.md index bcc8340210..5fcdde4846 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,62 @@ +# Synapse 1.119.0rc2 (2024-11-11) + +Note that due to packaging issues there was no v1.119.0rc1. + +### Python 3.8 support dropped + +Python 3.8 is [end-of-life](https://devguide.python.org/versions/) and is no longer supported by Synapse. The minimum supported Python version is now 3.9. + +If you are running Synapse with Python 3.8, please upgrade to Python 3.9 (or greater) before upgrading Synapse. + +### Features + +- Support [MSC4151](https://github.com/matrix-org/matrix-spec-proposals/pull/4151)'s stable report room API. ([\#17374](https://github.com/element-hq/synapse/issues/17374)) +- Add experimental support for [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222) (Adding `state_after` to sync v2). ([\#17888](https://github.com/element-hq/synapse/issues/17888)) + +### Bugfixes + +- Fix bug with sliding sync where `$LAZY`-loading room members would not return `required_state` membership in incremental syncs. ([\#17809](https://github.com/element-hq/synapse/issues/17809)) +- Check if user has membership in a room before tagging it. Contributed by Lama Alosaimi. ([\#17839](https://github.com/element-hq/synapse/issues/17839)) +- Fix a bug in the admin redact endpoint where the background task would not run if a worker was specified in + the config option `run_background_tasks_on`. ([\#17847](https://github.com/element-hq/synapse/issues/17847)) +- Fix bug where some presence and typing timeouts can expire early. ([\#17850](https://github.com/element-hq/synapse/issues/17850)) +- Fix detection when the built Rust library was outdated when using source installations. ([\#17861](https://github.com/element-hq/synapse/issues/17861)) +- Fix a long-standing bug in Synapse which could cause one-time keys to be issued in the incorrect order, causing message decryption failures. ([\#17903](https://github.com/element-hq/synapse/pull/17903)) +- Fix experimental support for [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222) (Adding `state_after` to sync v2) where we would return the full state on incremental syncs when using lazy loaded members and there were no new events in the timeline. ([\#17915](https://github.com/element-hq/synapse/pull/17915)) + +### Internal Changes + +- Remove support for python 3.8. ([\#17908](https://github.com/element-hq/synapse/issues/17908)) +- Add a test for downloading and thumbnailing a CMYK JPEG. ([\#17786](https://github.com/element-hq/synapse/issues/17786)) +- Refactor database calls to remove `Generator` usage. ([\#17813](https://github.com/element-hq/synapse/issues/17813), [\#17814](https://github.com/element-hq/synapse/issues/17814), [\#17815](https://github.com/element-hq/synapse/issues/17815), [\#17816](https://github.com/element-hq/synapse/issues/17816), [\#17817](https://github.com/element-hq/synapse/issues/17817), [\#17818](https://github.com/element-hq/synapse/issues/17818), [\#17890](https://github.com/element-hq/synapse/issues/17890)) +- Include the destination in the error of 'Destination mismatch' on federation requests. ([\#17830](https://github.com/element-hq/synapse/issues/17830)) +- The nix flake inside the repository no longer tracks nixpkgs/master to not catch the latest bugs from a PR merged 5 minutes ago. ([\#17852](https://github.com/element-hq/synapse/issues/17852)) +- Minor speed-up of sliding sync by computing extensions results in parallel. ([\#17884](https://github.com/element-hq/synapse/issues/17884)) +- Bump the default Python version in the Synapse Dockerfile from 3.11 -> 3.12. ([\#17887](https://github.com/element-hq/synapse/issues/17887)) +- Remove usage of internal header encoding API. ([\#17894](https://github.com/element-hq/synapse/issues/17894)) +- Use unique name for each os.arch variant when uploading Wheel artifacts. ([\#17905](https://github.com/element-hq/synapse/issues/17905)) +- Fix tests to run with latest Twisted. ([\#17906](https://github.com/element-hq/synapse/pull/17906), [\#17907](https://github.com/element-hq/synapse/pull/17907), [\#17911](https://github.com/element-hq/synapse/pull/17911)) +- Update version constraint to allow the latest poetry-core 1.9.1. ([\#17902](https://github.com/element-hq/synapse/pull/17902)) +- Update the portdb CI to use Python 3.13 and Postgres 17 as latest dependencies. ([\#17909](https://github.com/element-hq/synapse/pull/17909)) +- Add an index to `current_state_delta_stream` table. ([\#17912](https://github.com/element-hq/synapse/issues/17912)) +- Fix building and attaching release artifacts during the release process. ([\#17921](https://github.com/element-hq/synapse/issues/17921)) + +### Updates to locked dependencies + +* Bump actions/download-artifact & actions/upload-artifact from 3 to 4 in /.github/workflows. ([\#17657](https://github.com/element-hq/synapse/issues/17657)) +* Bump anyhow from 1.0.89 to 1.0.92. ([\#17858](https://github.com/element-hq/synapse/issues/17858), [\#17876](https://github.com/element-hq/synapse/issues/17876), [\#17901](https://github.com/element-hq/synapse/issues/17901)) +* Bump bytes from 1.7.2 to 1.8.0. ([\#17877](https://github.com/element-hq/synapse/issues/17877)) +* Bump cryptography from 43.0.1 to 43.0.3. ([\#17853](https://github.com/element-hq/synapse/issues/17853)) +* Bump mypy-zope from 1.0.7 to 1.0.8. ([\#17898](https://github.com/element-hq/synapse/issues/17898)) +* Bump phonenumbers from 8.13.47 to 8.13.49. ([\#17880](https://github.com/element-hq/synapse/issues/17880), [\#17899](https://github.com/element-hq/synapse/issues/17899)) +* Bump python-multipart from 0.0.12 to 0.0.16. ([\#17879](https://github.com/element-hq/synapse/issues/17879)) +* Bump regex from 1.11.0 to 1.11.1. ([\#17874](https://github.com/element-hq/synapse/issues/17874)) +* Bump ruff from 0.6.9 to 0.7.2. ([\#17868](https://github.com/element-hq/synapse/issues/17868), [\#17897](https://github.com/element-hq/synapse/issues/17897)) +* Bump serde from 1.0.210 to 1.0.214. ([\#17875](https://github.com/element-hq/synapse/issues/17875), [\#17900](https://github.com/element-hq/synapse/issues/17900)) +* Bump serde_json from 1.0.128 to 1.0.132. ([\#17857](https://github.com/element-hq/synapse/issues/17857)) +* Bump types-psycopg2 from 2.9.21.20240819 to 2.9.21.20241019. ([\#17855](https://github.com/element-hq/synapse/issues/17855)) +* Bump types-setuptools from 75.1.0.20241014 to 75.2.0.20241019. ([\#17856](https://github.com/element-hq/synapse/issues/17856)) + # Synapse 1.118.0 (2024-10-29) No significant changes since 1.118.0rc1. diff --git a/changelog.d/17374.feature b/changelog.d/17374.feature deleted file mode 100644 index 3321f18947..0000000000 --- a/changelog.d/17374.feature +++ /dev/null @@ -1 +0,0 @@ -Support [MSC4151](https://github.com/matrix-org/matrix-spec-proposals/pull/4151)'s stable report room API. \ No newline at end of file diff --git a/changelog.d/17657.misc b/changelog.d/17657.misc deleted file mode 100644 index aff558adf7..0000000000 --- a/changelog.d/17657.misc +++ /dev/null @@ -1 +0,0 @@ -Bump actions/download-artifact and actions/upload-artifact from v3 -> v4. diff --git a/changelog.d/17786.misc b/changelog.d/17786.misc deleted file mode 100644 index 59eb3f4dbd..0000000000 --- a/changelog.d/17786.misc +++ /dev/null @@ -1 +0,0 @@ -Add a test for downloading and thumbnailing a CMYK JPEG. \ No newline at end of file diff --git a/changelog.d/17809.bugfix b/changelog.d/17809.bugfix deleted file mode 100644 index e244a36bd3..0000000000 --- a/changelog.d/17809.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug with sliding sync where `$LAZY`-loading room members would not return `required_state` membership in incremental syncs. diff --git a/changelog.d/17813.misc b/changelog.d/17813.misc deleted file mode 100644 index f8676aee59..0000000000 --- a/changelog.d/17813.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor database calls to remove `Generator` usage. diff --git a/changelog.d/17814.misc b/changelog.d/17814.misc deleted file mode 100644 index f8676aee59..0000000000 --- a/changelog.d/17814.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor database calls to remove `Generator` usage. diff --git a/changelog.d/17815.misc b/changelog.d/17815.misc deleted file mode 100644 index f8676aee59..0000000000 --- a/changelog.d/17815.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor database calls to remove `Generator` usage. diff --git a/changelog.d/17816.misc b/changelog.d/17816.misc deleted file mode 100644 index f8676aee59..0000000000 --- a/changelog.d/17816.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor database calls to remove `Generator` usage. diff --git a/changelog.d/17817.misc b/changelog.d/17817.misc deleted file mode 100644 index f8676aee59..0000000000 --- a/changelog.d/17817.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor database calls to remove `Generator` usage. diff --git a/changelog.d/17818.misc b/changelog.d/17818.misc deleted file mode 100644 index f8676aee59..0000000000 --- a/changelog.d/17818.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor database calls to remove `Generator` usage. diff --git a/changelog.d/17830.misc b/changelog.d/17830.misc deleted file mode 100644 index b2cd91c0ad..0000000000 --- a/changelog.d/17830.misc +++ /dev/null @@ -1 +0,0 @@ -Include the destination in the error of 'Destination mismatch' on federation requests. diff --git a/changelog.d/17839.bugfix b/changelog.d/17839.bugfix deleted file mode 100644 index 57667a6df5..0000000000 --- a/changelog.d/17839.bugfix +++ /dev/null @@ -1 +0,0 @@ -Check if user has membership in a room before tagging it. Contributed by Lama Alosaimi. \ No newline at end of file diff --git a/changelog.d/17847.bugfix b/changelog.d/17847.bugfix deleted file mode 100644 index 0ba39df94d..0000000000 --- a/changelog.d/17847.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug in the admin redact endpoint where the background task would not run if a worker was specified in -the config option `run_background_tasks_on`. \ No newline at end of file diff --git a/changelog.d/17850.bugfix b/changelog.d/17850.bugfix deleted file mode 100644 index 8ea99c4ef9..0000000000 --- a/changelog.d/17850.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug when some presence and typing timeouts can expire early. \ No newline at end of file diff --git a/changelog.d/17852.misc b/changelog.d/17852.misc deleted file mode 100644 index b1b7ac9734..0000000000 --- a/changelog.d/17852.misc +++ /dev/null @@ -1 +0,0 @@ -The nix flake inside the repository no longer tracks nixpkgs/master to not catch the latest bugs from a PR merged 5 minutes ago. diff --git a/changelog.d/17861.bugfix b/changelog.d/17861.bugfix deleted file mode 100644 index abee7a30f7..0000000000 --- a/changelog.d/17861.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix detection when the built Rust library was outdated when using source installations. diff --git a/changelog.d/17884.misc b/changelog.d/17884.misc deleted file mode 100644 index 9dfa13f853..0000000000 --- a/changelog.d/17884.misc +++ /dev/null @@ -1 +0,0 @@ -Minor speed-up of sliding sync by computing extensions results in parallel. diff --git a/changelog.d/17887.misc b/changelog.d/17887.misc deleted file mode 100644 index 6be32caee6..0000000000 --- a/changelog.d/17887.misc +++ /dev/null @@ -1 +0,0 @@ -Bump the default Python version in the Synapse Dockerfile from 3.11 -> 3.12. \ No newline at end of file diff --git a/changelog.d/17888.feature b/changelog.d/17888.feature deleted file mode 100644 index 3ede8886ab..0000000000 --- a/changelog.d/17888.feature +++ /dev/null @@ -1 +0,0 @@ -Add experimental support for [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222). diff --git a/changelog.d/17890.misc b/changelog.d/17890.misc deleted file mode 100644 index f8676aee59..0000000000 --- a/changelog.d/17890.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor database calls to remove `Generator` usage. diff --git a/changelog.d/17894.misc b/changelog.d/17894.misc deleted file mode 100644 index dc1a7577ab..0000000000 --- a/changelog.d/17894.misc +++ /dev/null @@ -1 +0,0 @@ -Remove usage of internal header encoding API. diff --git a/changelog.d/17902.misc b/changelog.d/17902.misc deleted file mode 100644 index f094f57c2f..0000000000 --- a/changelog.d/17902.misc +++ /dev/null @@ -1 +0,0 @@ -Update version constraint to allow the latest poetry-core 1.9.1. diff --git a/changelog.d/17903.bugfix b/changelog.d/17903.bugfix deleted file mode 100644 index a4d02fc983..0000000000 --- a/changelog.d/17903.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug in Synapse which could cause one-time keys to be issued in the incorrect order, causing message decryption failures. diff --git a/changelog.d/17905.misc b/changelog.d/17905.misc deleted file mode 100644 index 32ef50dbac..0000000000 --- a/changelog.d/17905.misc +++ /dev/null @@ -1 +0,0 @@ -Use unique name for each os.arch variant when uploading Wheel artifacts. diff --git a/changelog.d/17906.bugfix b/changelog.d/17906.bugfix deleted file mode 100644 index f38ce6a590..0000000000 --- a/changelog.d/17906.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix tests to run with latest Twisted. diff --git a/changelog.d/17907.bugfix b/changelog.d/17907.bugfix deleted file mode 100644 index f38ce6a590..0000000000 --- a/changelog.d/17907.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix tests to run with latest Twisted. diff --git a/changelog.d/17908.misc b/changelog.d/17908.misc deleted file mode 100644 index 8f17729148..0000000000 --- a/changelog.d/17908.misc +++ /dev/null @@ -1 +0,0 @@ -Remove support for python 3.8. diff --git a/changelog.d/17909.misc b/changelog.d/17909.misc deleted file mode 100644 index f826aa7948..0000000000 --- a/changelog.d/17909.misc +++ /dev/null @@ -1 +0,0 @@ -Update the portdb CI to use Python 3.13 and Postgres 17 as latest dependencies. \ No newline at end of file diff --git a/changelog.d/17911.bugfix b/changelog.d/17911.bugfix deleted file mode 100644 index f38ce6a590..0000000000 --- a/changelog.d/17911.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix tests to run with latest Twisted. diff --git a/changelog.d/17915.bugfix b/changelog.d/17915.bugfix deleted file mode 100644 index a5d82e486d..0000000000 --- a/changelog.d/17915.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix experimental support for [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222) where we would return the full state on incremental syncs when using lazy loaded members and there were no new events in the timeline. diff --git a/debian/changelog b/debian/changelog index 384887888f..10ca8fbb20 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +matrix-synapse-py3 (1.119.0~rc2) stable; urgency=medium + + * New Synapse release 1.119.0rc2. + + -- Synapse Packaging team Mon, 11 Nov 2024 14:33:02 +0000 + +matrix-synapse-py3 (1.119.0~rc1) stable; urgency=medium + + * New Synapse release 1.119.0rc1. + + -- Synapse Packaging team Wed, 06 Nov 2024 08:59:43 -0700 + matrix-synapse-py3 (1.118.0) stable; urgency=medium * New Synapse release 1.118.0. diff --git a/pyproject.toml b/pyproject.toml index 90bd688207..5c34c19565 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.118.0" +version = "1.119.0rc2" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" diff --git a/synapse/storage/databases/main/state_deltas.py b/synapse/storage/databases/main/state_deltas.py index ba52fff652..117ee89d0a 100644 --- a/synapse/storage/databases/main/state_deltas.py +++ b/synapse/storage/databases/main/state_deltas.py @@ -20,18 +20,26 @@ # import logging -from typing import List, Optional, Tuple +from typing import TYPE_CHECKING, List, Optional, Tuple import attr from synapse.logging.opentracing import trace from synapse.storage._base import SQLBaseStore -from synapse.storage.database import LoggingTransaction, make_in_list_sql_clause +from synapse.storage.database import ( + DatabasePool, + LoggingDatabaseConnection, + LoggingTransaction, + make_in_list_sql_clause, +) from synapse.storage.databases.main.stream import _filter_results_by_stream from synapse.types import RoomStreamToken, StrCollection from synapse.util.caches.stream_change_cache import StreamChangeCache from synapse.util.iterutils import batch_iter +if TYPE_CHECKING: + from synapse.server import HomeServer + logger = logging.getLogger(__name__) @@ -54,6 +62,21 @@ class StateDeltasStore(SQLBaseStore): # attribute. TODO: can we get static analysis to enforce this? _curr_state_delta_stream_cache: StreamChangeCache + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + super().__init__(database, db_conn, hs) + + self.db_pool.updates.register_background_index_update( + update_name="current_state_delta_stream_room_index", + index_name="current_state_delta_stream_room_idx", + table="current_state_delta_stream", + columns=("room_id", "stream_id"), + ) + async def get_partial_current_state_deltas( self, prev_stream_id: int, max_stream_id: int ) -> Tuple[int, List[StateDelta]]: diff --git a/synapse/storage/schema/main/delta/88/04_current_state_delta_index.sql b/synapse/storage/schema/main/delta/88/04_current_state_delta_index.sql new file mode 100644 index 0000000000..ad54302a8f --- /dev/null +++ b/synapse/storage/schema/main/delta/88/04_current_state_delta_index.sql @@ -0,0 +1,18 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2024 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + + +-- Add an index on (user_id, device_id, algorithm, ts_added_ms) on e2e_one_time_keys_json, so that OTKs can +-- efficiently be issued in the same order they were uploaded. +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (8804, 'current_state_delta_stream_room_index', '{}');