Merge branch 'develop' into matrix-org-hotfixes

changelog.d/16857.doc

@@ -0,0 +1 @@
+Fix a small typo in the Rooms section of the Admin API documentation. Contributed by @RainerZufall187.

changelog.d/16881.feature

@@ -0,0 +1 @@
+A metric was added for emails sent by Synapse, broken down by type: `synapse_emails_sent_total`. Contributed by Remi Rampin.

changelog.d/16903.bugfix

@@ -0,0 +1 @@
+Fix performance issue when joining very large rooms that can cause the server to lock up. Introduced in v1.100.0.

changelog.d/16910.misc

@@ -0,0 +1 @@
+Fix incorrect docker hub link in release script.

changelog.d/16927.bugfix

@@ -0,0 +1 @@
+Always prefer unthreaded receipt when >1 exist ([MSC4102](https://github.com/matrix-org/matrix-spec-proposals/pull/4102)).

docs/admin_api/rooms.md

@@ -913,7 +913,7 @@ With all that being said, if you still want to try and recover the room:
    them handle rejoining themselves.
 
 4. If `new_room_user_id` was given, a 'Content Violation' will have been
-   created. Consider whether you want to delete that roomm.
+   created. Consider whether you want to delete that room.
 
 # Make Room Admin API
 

scripts-dev/release.py

@@ -660,7 +660,7 @@ def _announce() -> None:
 Hi everyone. Synapse {current_version} has just been released.
 
 [notes](https://github.com/element-hq/synapse/releases/tag/{tag_name}) | \
-[docker](https://hub.docker.com/r/vectorim/synapse/tags?name={tag_name}) | \
+[docker](https://hub.docker.com/r/matrixdotorg/synapse/tags?name={tag_name}) | \
 [debs](https://packages.matrix.org/debian/) | \
 [pypi](https://pypi.org/project/matrix-synapse/{current_version}/)"""
     )

synapse/handlers/federation_event.py

@@ -1757,17 +1757,25 @@ class FederationEventHandler:
 
             events_and_contexts_to_persist.append((event, context))
 
-        for event in sorted_auth_events:
+        for i, event in enumerate(sorted_auth_events):
             await prep(event)
 
-        await self.persist_events_and_notify(
+            # The above function is typically not async, and so won't yield to
-            room_id,
+            # the reactor. For large rooms let's yield to the reactor
-            events_and_contexts_to_persist,
+            # occasionally to ensure we don't block other work.
-            # Mark these events backfilled as they're historic events that will
+            if (i + 1) % 1000 == 0:
-            # eventually be backfilled. For example, missing events we fetch
+                await self._clock.sleep(0)
-            # during backfill should be marked as backfilled as well.
+
-            backfilled=True,
+        # Also persist the new event in batches for similar reasons as above.
-        )
+        for batch in batch_iter(events_and_contexts_to_persist, 1000):
+            await self.persist_events_and_notify(
+                room_id,
+                batch,
+                # Mark these events as backfilled as they're historic events that will
+                # eventually be backfilled. For example, missing events we fetch
+                # during backfill should be marked as backfilled as well.
+                backfilled=True,
+            )
 
     @trace
     async def _check_event_auth(

synapse/push/mailer.py

@@ -26,6 +26,7 @@ from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, TypeVar
 import bleach
 import jinja2
 from markupsafe import Markup
+from prometheus_client import Counter
 
 from synapse.api.constants import EventTypes, Membership, RoomTypes
 from synapse.api.errors import StoreError
@@ -56,6 +57,12 @@ logger = logging.getLogger(__name__)
 
 T = TypeVar("T")
 
+emails_sent_counter = Counter(
+    "synapse_emails_sent_total",
+    "Emails sent by type",
+    ["type"],
+)
+
 
 CONTEXT_BEFORE = 1
 CONTEXT_AFTER = 1
@@ -130,6 +137,8 @@ class Mailer:
 
         logger.info("Created Mailer for app_name %s" % app_name)
 
+    emails_sent_counter.labels("password_reset")
+
     async def send_password_reset_mail(
         self, email_address: str, token: str, client_secret: str, sid: str
     ) -> None:
@@ -153,6 +162,8 @@ class Mailer:
 
         template_vars: TemplateVars = {"link": link}
 
+        emails_sent_counter.labels("password_reset").inc()
+
         await self.send_email(
             email_address,
             self.email_subjects.password_reset
@@ -160,6 +171,8 @@ class Mailer:
             template_vars,
         )
 
+    emails_sent_counter.labels("registration")
+
     async def send_registration_mail(
         self, email_address: str, token: str, client_secret: str, sid: str
     ) -> None:
@@ -183,6 +196,8 @@ class Mailer:
 
         template_vars: TemplateVars = {"link": link}
 
+        emails_sent_counter.labels("registration").inc()
+
         await self.send_email(
             email_address,
             self.email_subjects.email_validation
@@ -190,6 +205,8 @@ class Mailer:
             template_vars,
         )
 
+    emails_sent_counter.labels("add_threepid")
+
     async def send_add_threepid_mail(
         self, email_address: str, token: str, client_secret: str, sid: str
     ) -> None:
@@ -214,6 +231,8 @@ class Mailer:
 
         template_vars: TemplateVars = {"link": link}
 
+        emails_sent_counter.labels("add_threepid").inc()
+
         await self.send_email(
             email_address,
             self.email_subjects.email_validation
@@ -221,6 +240,8 @@ class Mailer:
             template_vars,
         )
 
+    emails_sent_counter.labels("notification")
+
     async def send_notification_mail(
         self,
         app_id: str,
@@ -315,6 +336,8 @@ class Mailer:
             "reason": reason,
         }
 
+        emails_sent_counter.labels("notification").inc()
+
         await self.send_email(
             email_address, summary_text, template_vars, unsubscribe_link
         )

synapse/storage/databases/main/receipts.py

@@ -472,9 +472,24 @@ class ReceiptsWorkerStore(SQLBaseStore):
             event_entry = room_event["content"].setdefault(event_id, {})
             receipt_type_dict = event_entry.setdefault(receipt_type, {})
 
-            receipt_type_dict[user_id] = db_to_json(data)
+            # MSC4102: always replace threaded receipts with unthreaded ones if there is a clash.
-            if thread_id:
+            # Specifically:
-                receipt_type_dict[user_id]["thread_id"] = thread_id
+            # - if there is no existing receipt, great, set the data.
+            # - if there is an existing receipt, is it threaded (thread_id present)?
+            #    YES: replace if this receipt has no thread id. NO: do not replace.
+            # This means we will drop some receipts, but MSC4102 is designed to drop semantically
+            # meaningless receipts, so this is okay. Previously, we would drop meaningful data!
+            receipt_data = db_to_json(data)
+            if user_id in receipt_type_dict:  # existing receipt
+                # is the existing receipt threaded and we are currently processing an unthreaded one?
+                if "thread_id" in receipt_type_dict[user_id] and not thread_id:
+                    receipt_type_dict[
+                        user_id
+                    ] = receipt_data  # replace with unthreaded one
+            else:  # receipt does not exist, just set it
+                receipt_type_dict[user_id] = receipt_data
+                if thread_id:
+                    receipt_type_dict[user_id]["thread_id"] = thread_id
 
         results = {
             room_id: [results[room_id]] if room_id in results else []