From 324525f40ca4df19c43971ca82db0d3478114885 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Mon, 20 Aug 2018 23:54:49 +1000
Subject: [PATCH 1/7] Port over enough to get some sytests running on Python 3
 (#3668)

---
 changelog.d/3668.misc                    |  1 +
 synapse/api/auth.py                      | 14 +++---
 synapse/api/ratelimiting.py              |  2 +-
 synapse/config/logger.py                 |  3 +-
 synapse/http/servlet.py                  | 56 ++++++++++++++++++------
 synapse/http/site.py                     |  4 +-
 synapse/rest/client/transactions.py      |  2 +-
 synapse/rest/media/v1/upload_resource.py | 10 ++---
 synapse/secrets.py                       |  2 +-
 synapse/util/logutils.py                 | 10 ++++-
 synapse/util/stringutils.py              | 15 +++++++
 synapse/util/versionstring.py            | 12 ++---
 12 files changed, 91 insertions(+), 40 deletions(-)
 create mode 100644 changelog.d/3668.misc

diff --git a/changelog.d/3668.misc b/changelog.d/3668.misc
new file mode 100644
index 0000000000..f9265addf1
--- /dev/null
+++ b/changelog.d/3668.misc
@@ -0,0 +1 @@
+Port over enough to Python 3 to allow the sytests to start.
diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 022211e34e..6502a6be7b 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -211,7 +211,7 @@ class Auth(object):
             user_agent = request.requestHeaders.getRawHeaders(
                 b"User-Agent",
                 default=[b""]
-            )[0]
+            )[0].decode('ascii', 'surrogateescape')
             if user and access_token and ip_addr:
                 yield self.store.insert_client_ip(
                     user_id=user.to_string(),
@@ -682,7 +682,7 @@ class Auth(object):
         Returns:
             bool: False if no access_token was given, True otherwise.
         """
-        query_params = request.args.get("access_token")
+        query_params = request.args.get(b"access_token")
         auth_headers = request.requestHeaders.getRawHeaders(b"Authorization")
         return bool(query_params) or bool(auth_headers)
 
@@ -698,7 +698,7 @@ class Auth(object):
                 401 since some of the old clients depended on auth errors returning
                 403.
         Returns:
-            str: The access_token
+            unicode: The access_token
         Raises:
             AuthError: If there isn't an access_token in the request.
         """
@@ -720,9 +720,9 @@ class Auth(object):
                     "Too many Authorization headers.",
                     errcode=Codes.MISSING_TOKEN,
                 )
-            parts = auth_headers[0].split(" ")
-            if parts[0] == "Bearer" and len(parts) == 2:
-                return parts[1]
+            parts = auth_headers[0].split(b" ")
+            if parts[0] == b"Bearer" and len(parts) == 2:
+                return parts[1].decode('ascii')
             else:
                 raise AuthError(
                     token_not_found_http_status,
@@ -738,7 +738,7 @@ class Auth(object):
                     errcode=Codes.MISSING_TOKEN
                 )
 
-            return query_params[0]
+            return query_params[0].decode('ascii')
 
     @defer.inlineCallbacks
     def check_in_room_or_world_readable(self, room_id, user_id):
diff --git a/synapse/api/ratelimiting.py b/synapse/api/ratelimiting.py
index 06cc8d90b8..3bb5b3da37 100644
--- a/synapse/api/ratelimiting.py
+++ b/synapse/api/ratelimiting.py
@@ -72,7 +72,7 @@ class Ratelimiter(object):
         return allowed, time_allowed
 
     def prune_message_counts(self, time_now_s):
-        for user_id in self.message_counts.keys():
+        for user_id in list(self.message_counts.keys()):
             message_count, time_start, msg_rate_hz = (
                 self.message_counts[user_id]
             )
diff --git a/synapse/config/logger.py b/synapse/config/logger.py
index cfc20dcccf..3f187adfc8 100644
--- a/synapse/config/logger.py
+++ b/synapse/config/logger.py
@@ -168,7 +168,8 @@ def setup_logging(config, use_worker_options=False):
         if log_file:
             # TODO: Customisable file size / backup count
             handler = logging.handlers.RotatingFileHandler(
-                log_file, maxBytes=(1000 * 1000 * 100), backupCount=3
+                log_file, maxBytes=(1000 * 1000 * 100), backupCount=3,
+                encoding='utf8'
             )
 
             def sighup(signum, stack):
diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py
index 69f7085291..a1e4b88e6d 100644
--- a/synapse/http/servlet.py
+++ b/synapse/http/servlet.py
@@ -29,7 +29,7 @@ def parse_integer(request, name, default=None, required=False):
 
     Args:
         request: the twisted HTTP request.
-        name (str): the name of the query parameter.
+        name (bytes/unicode): the name of the query parameter.
         default (int|None): value to use if the parameter is absent, defaults
             to None.
         required (bool): whether to raise a 400 SynapseError if the
@@ -46,6 +46,10 @@ def parse_integer(request, name, default=None, required=False):
 
 
 def parse_integer_from_args(args, name, default=None, required=False):
+
+    if not isinstance(name, bytes):
+        name = name.encode('ascii')
+
     if name in args:
         try:
             return int(args[name][0])
@@ -65,7 +69,7 @@ def parse_boolean(request, name, default=None, required=False):
 
     Args:
         request: the twisted HTTP request.
-        name (str): the name of the query parameter.
+        name (bytes/unicode): the name of the query parameter.
         default (bool|None): value to use if the parameter is absent, defaults
             to None.
         required (bool): whether to raise a 400 SynapseError if the
@@ -83,11 +87,15 @@ def parse_boolean(request, name, default=None, required=False):
 
 
 def parse_boolean_from_args(args, name, default=None, required=False):
+
+    if not isinstance(name, bytes):
+        name = name.encode('ascii')
+
     if name in args:
         try:
             return {
-                "true": True,
-                "false": False,
+                b"true": True,
+                b"false": False,
             }[args[name][0]]
         except Exception:
             message = (
@@ -104,21 +112,29 @@ def parse_boolean_from_args(args, name, default=None, required=False):
 
 
 def parse_string(request, name, default=None, required=False,
-                 allowed_values=None, param_type="string"):
-    """Parse a string parameter from the request query string.
+                 allowed_values=None, param_type="string", encoding='ascii'):
+    """
+    Parse a string parameter from the request query string.
+
+    If encoding is not None, the content of the query param will be
+    decoded to Unicode using the encoding, otherwise it will be encoded
 
     Args:
         request: the twisted HTTP request.
-        name (str): the name of the query parameter.
-        default (str|None): value to use if the parameter is absent, defaults
-            to None.
+        name (bytes/unicode): the name of the query parameter.
+        default (bytes/unicode|None): value to use if the parameter is absent,
+            defaults to None. Must be bytes if encoding is None.
         required (bool): whether to raise a 400 SynapseError if the
             parameter is absent, defaults to False.
-        allowed_values (list[str]): List of allowed values for the string,
-            or None if any value is allowed, defaults to None
+        allowed_values (list[bytes/unicode]): List of allowed values for the
+            string, or None if any value is allowed, defaults to None. Must be
+            the same type as name, if given.
+        encoding: The encoding to decode the name to, and decode the string
+            content with.
 
     Returns:
-        str|None: A string value or the default.
+        bytes/unicode|None: A string value or the default. Unicode if encoding
+        was given, bytes otherwise.
 
     Raises:
         SynapseError if the parameter is absent and required, or if the
@@ -126,14 +142,22 @@ def parse_string(request, name, default=None, required=False,
             is not one of those allowed values.
     """
     return parse_string_from_args(
-        request.args, name, default, required, allowed_values, param_type,
+        request.args, name, default, required, allowed_values, param_type, encoding
     )
 
 
 def parse_string_from_args(args, name, default=None, required=False,
-                           allowed_values=None, param_type="string"):
+                           allowed_values=None, param_type="string", encoding='ascii'):
+
+    if not isinstance(name, bytes):
+        name = name.encode('ascii')
+
     if name in args:
         value = args[name][0]
+
+        if encoding:
+            value = value.decode(encoding)
+
         if allowed_values is not None and value not in allowed_values:
             message = "Query parameter %r must be one of [%s]" % (
                 name, ", ".join(repr(v) for v in allowed_values)
@@ -146,6 +170,10 @@ def parse_string_from_args(args, name, default=None, required=False,
             message = "Missing %s query parameter %r" % (param_type, name)
             raise SynapseError(400, message, errcode=Codes.MISSING_PARAM)
         else:
+
+            if encoding and isinstance(default, bytes):
+                return default.decode(encoding)
+
             return default
 
 
diff --git a/synapse/http/site.py b/synapse/http/site.py
index f5a8f78406..ad2a98468e 100644
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -235,7 +235,7 @@ class SynapseRequest(Request):
         # need to decode as it could be raw utf-8 bytes
         # from a IDN servname in an auth header
         authenticated_entity = self.authenticated_entity
-        if authenticated_entity is not None:
+        if authenticated_entity is not None and isinstance(authenticated_entity, bytes):
             authenticated_entity = authenticated_entity.decode("utf-8", "replace")
 
         # ...or could be raw utf-8 bytes in the User-Agent header.
@@ -328,7 +328,7 @@ class SynapseSite(Site):
         proxied = config.get("x_forwarded", False)
         self.requestFactory = SynapseRequestFactory(self, proxied)
         self.access_logger = logging.getLogger(logger_name)
-        self.server_version_string = server_version_string
+        self.server_version_string = server_version_string.encode('ascii')
 
     def log(self, request):
         pass
diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py
index 511e96ab00..48c17f1b6d 100644
--- a/synapse/rest/client/transactions.py
+++ b/synapse/rest/client/transactions.py
@@ -53,7 +53,7 @@ class HttpTransactionCache(object):
             str: A transaction key
         """
         token = self.auth.get_access_token_from_request(request)
-        return request.path + "/" + token
+        return request.path.decode('utf8') + "/" + token
 
     def fetch_or_execute_request(self, request, fn, *args, **kwargs):
         """A helper function for fetch_or_execute which extracts
diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py
index 9b22d204a6..c1240e1963 100644
--- a/synapse/rest/media/v1/upload_resource.py
+++ b/synapse/rest/media/v1/upload_resource.py
@@ -55,7 +55,7 @@ class UploadResource(Resource):
         requester = yield self.auth.get_user_by_req(request)
         # TODO: The checks here are a bit late. The content will have
         # already been uploaded to a tmp file at this point
-        content_length = request.getHeader("Content-Length")
+        content_length = request.getHeader(b"Content-Length").decode('ascii')
         if content_length is None:
             raise SynapseError(
                 msg="Request must specify a Content-Length", code=400
@@ -66,10 +66,10 @@ class UploadResource(Resource):
                 code=413,
             )
 
-        upload_name = parse_string(request, "filename")
+        upload_name = parse_string(request, b"filename", encoding=None)
         if upload_name:
             try:
-                upload_name = upload_name.decode('UTF-8')
+                upload_name = upload_name.decode('utf8')
             except UnicodeDecodeError:
                 raise SynapseError(
                     msg="Invalid UTF-8 filename parameter: %r" % (upload_name),
@@ -78,8 +78,8 @@ class UploadResource(Resource):
 
         headers = request.requestHeaders
 
-        if headers.hasHeader("Content-Type"):
-            media_type = headers.getRawHeaders(b"Content-Type")[0]
+        if headers.hasHeader(b"Content-Type"):
+            media_type = headers.getRawHeaders(b"Content-Type")[0].decode('ascii')
         else:
             raise SynapseError(
                 msg="Upload request missing 'Content-Type'",
diff --git a/synapse/secrets.py b/synapse/secrets.py
index f05e9ea535..f6280f951c 100644
--- a/synapse/secrets.py
+++ b/synapse/secrets.py
@@ -38,4 +38,4 @@ else:
             return os.urandom(nbytes)
 
         def token_hex(self, nbytes=32):
-            return binascii.hexlify(self.token_bytes(nbytes))
+            return binascii.hexlify(self.token_bytes(nbytes)).decode('ascii')
diff --git a/synapse/util/logutils.py b/synapse/util/logutils.py
index 62a00189cc..ef31458226 100644
--- a/synapse/util/logutils.py
+++ b/synapse/util/logutils.py
@@ -20,6 +20,8 @@ import time
 from functools import wraps
 from inspect import getcallargs
 
+from six import PY3
+
 _TIME_FUNC_ID = 0
 
 
@@ -28,8 +30,12 @@ def _log_debug_as_f(f, msg, msg_args):
     logger = logging.getLogger(name)
 
     if logger.isEnabledFor(logging.DEBUG):
-        lineno = f.func_code.co_firstlineno
-        pathname = f.func_code.co_filename
+        if PY3:
+            lineno = f.__code__.co_firstlineno
+            pathname = f.__code__.co_filename
+        else:
+            lineno = f.func_code.co_firstlineno
+            pathname = f.func_code.co_filename
 
         record = logging.LogRecord(
             name=name,
diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py
index 43d9db67ec..6f318c6a29 100644
--- a/synapse/util/stringutils.py
+++ b/synapse/util/stringutils.py
@@ -16,6 +16,7 @@
 import random
 import string
 
+from six import PY3
 from six.moves import range
 
 _string_with_symbols = (
@@ -34,6 +35,17 @@ def random_string_with_symbols(length):
 
 
 def is_ascii(s):
+
+    if PY3:
+        if isinstance(s, bytes):
+            try:
+                s.decode('ascii').encode('ascii')
+            except UnicodeDecodeError:
+                return False
+            except UnicodeEncodeError:
+                return False
+            return True
+
     try:
         s.encode("ascii")
     except UnicodeEncodeError:
@@ -49,6 +61,9 @@ def to_ascii(s):
 
     If given None then will return None.
     """
+    if PY3:
+        return s
+
     if s is None:
         return None
 
diff --git a/synapse/util/versionstring.py b/synapse/util/versionstring.py
index 1fbcd41115..3baba3225a 100644
--- a/synapse/util/versionstring.py
+++ b/synapse/util/versionstring.py
@@ -30,7 +30,7 @@ def get_version_string(module):
                 ['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
                 stderr=null,
                 cwd=cwd,
-            ).strip()
+            ).strip().decode('ascii')
             git_branch = "b=" + git_branch
         except subprocess.CalledProcessError:
             git_branch = ""
@@ -40,7 +40,7 @@ def get_version_string(module):
                 ['git', 'describe', '--exact-match'],
                 stderr=null,
                 cwd=cwd,
-            ).strip()
+            ).strip().decode('ascii')
             git_tag = "t=" + git_tag
         except subprocess.CalledProcessError:
             git_tag = ""
@@ -50,7 +50,7 @@ def get_version_string(module):
                 ['git', 'rev-parse', '--short', 'HEAD'],
                 stderr=null,
                 cwd=cwd,
-            ).strip()
+            ).strip().decode('ascii')
         except subprocess.CalledProcessError:
             git_commit = ""
 
@@ -60,7 +60,7 @@ def get_version_string(module):
                 ['git', 'describe', '--dirty=' + dirty_string],
                 stderr=null,
                 cwd=cwd,
-            ).strip().endswith(dirty_string)
+            ).strip().decode('ascii').endswith(dirty_string)
 
             git_dirty = "dirty" if is_dirty else ""
         except subprocess.CalledProcessError:
@@ -77,8 +77,8 @@ def get_version_string(module):
                 "%s (%s)" % (
                     module.__version__, git_version,
                 )
-            ).encode("ascii")
+            )
     except Exception as e:
         logger.info("Failed to check for git repository: %s", e)
 
-    return module.__version__.encode("ascii")
+    return module.__version__

From 9a2f960736a1d1aeb6c9c094887253b280a59a37 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 21 Aug 2018 00:00:19 +1000
Subject: [PATCH 2/7] version

---
 synapse/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/__init__.py b/synapse/__init__.py
index a14d578e36..5175a1161d 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -17,4 +17,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.33.2"
+__version__ = "0.33.3rc1"

From 80bf7d35802003cee6e495421ec54c1c08bce525 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 21 Aug 2018 00:01:14 +1000
Subject: [PATCH 3/7] changelog

---
 CHANGES.md               | 64 ++++++++++++++++++++++++++++++++++++++++
 changelog.d/1491.feature |  1 -
 changelog.d/3184.feature |  1 -
 changelog.d/3423.misc    |  1 -
 changelog.d/3568.feature |  1 -
 changelog.d/3574.feature |  1 -
 changelog.d/3589.feature |  1 -
 changelog.d/3632.misc    |  1 -
 changelog.d/3633.feature |  1 -
 changelog.d/3647.misc    |  1 -
 changelog.d/3653.feature |  1 -
 changelog.d/3654.feature |  1 -
 changelog.d/3655.feature |  1 -
 changelog.d/3658.bugfix  |  1 -
 changelog.d/3660.misc    |  1 -
 changelog.d/3661.bugfix  |  1 -
 changelog.d/3662.feature |  1 -
 changelog.d/3664.feature |  1 -
 changelog.d/3668.misc    |  1 -
 changelog.d/3669.misc    |  1 -
 changelog.d/3670.feature |  1 -
 changelog.d/3676.bugfix  |  1 -
 changelog.d/3677.bugfix  |  1 -
 changelog.d/3678.misc    |  1 -
 changelog.d/3679.misc    |  1 -
 changelog.d/3681.bugfix  |  1 -
 changelog.d/3684.misc    |  1 -
 changelog.d/3687.feature |  1 -
 changelog.d/3689.bugfix  |  1 -
 changelog.d/3690.misc    |  1 -
 changelog.d/3692.bugfix  |  1 -
 changelog.d/3694.feature |  1 -
 changelog.d/3700.bugfix  |  1 -
 changelog.d/3701.bugfix  |  1 -
 changelog.d/3703.removal |  1 -
 changelog.d/3705.bugfix  |  1 -
 changelog.d/3707.misc    |  1 -
 changelog.d/3708.feature |  1 -
 changelog.d/3709.misc    |  1 -
 changelog.d/3710.bugfix  |  1 -
 changelog.d/3712.misc    |  1 -
 changelog.d/3713.bugfix  |  1 -
 changelog.d/3719.bugfix  |  1 -
 43 files changed, 64 insertions(+), 42 deletions(-)
 delete mode 100644 changelog.d/1491.feature
 delete mode 100644 changelog.d/3184.feature
 delete mode 100644 changelog.d/3423.misc
 delete mode 100644 changelog.d/3568.feature
 delete mode 100644 changelog.d/3574.feature
 delete mode 100644 changelog.d/3589.feature
 delete mode 100644 changelog.d/3632.misc
 delete mode 100644 changelog.d/3633.feature
 delete mode 100644 changelog.d/3647.misc
 delete mode 100644 changelog.d/3653.feature
 delete mode 100644 changelog.d/3654.feature
 delete mode 100644 changelog.d/3655.feature
 delete mode 100644 changelog.d/3658.bugfix
 delete mode 100644 changelog.d/3660.misc
 delete mode 100644 changelog.d/3661.bugfix
 delete mode 100644 changelog.d/3662.feature
 delete mode 100644 changelog.d/3664.feature
 delete mode 100644 changelog.d/3668.misc
 delete mode 100644 changelog.d/3669.misc
 delete mode 100644 changelog.d/3670.feature
 delete mode 100644 changelog.d/3676.bugfix
 delete mode 100644 changelog.d/3677.bugfix
 delete mode 100644 changelog.d/3678.misc
 delete mode 100644 changelog.d/3679.misc
 delete mode 100644 changelog.d/3681.bugfix
 delete mode 100644 changelog.d/3684.misc
 delete mode 100644 changelog.d/3687.feature
 delete mode 100644 changelog.d/3689.bugfix
 delete mode 100644 changelog.d/3690.misc
 delete mode 100644 changelog.d/3692.bugfix
 delete mode 100644 changelog.d/3694.feature
 delete mode 100644 changelog.d/3700.bugfix
 delete mode 100644 changelog.d/3701.bugfix
 delete mode 100644 changelog.d/3703.removal
 delete mode 100644 changelog.d/3705.bugfix
 delete mode 100644 changelog.d/3707.misc
 delete mode 100644 changelog.d/3708.feature
 delete mode 100644 changelog.d/3709.misc
 delete mode 100644 changelog.d/3710.bugfix
 delete mode 100644 changelog.d/3712.misc
 delete mode 100644 changelog.d/3713.bugfix
 delete mode 100644 changelog.d/3719.bugfix

diff --git a/CHANGES.md b/CHANGES.md
index a299110a6b..68ce5d1471 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,67 @@
+Synapse 0.33.3rc1 (2018-08-21)
+==============================
+
+Features
+--------
+
+- Add support for the SNI extension to federation TLS connections ([\#1491](https://github.com/matrix-org/synapse/issues/1491))
+- Add /_media/r0/config ([\#3184](https://github.com/matrix-org/synapse/issues/3184))
+- speed up /members API and add `at` and `membership` params as per MSC1227 ([\#3568](https://github.com/matrix-org/synapse/issues/3568))
+- implement `summary` block in /sync response as per MSC688 ([\#3574](https://github.com/matrix-org/synapse/issues/3574))
+- Add lazy-loading support to /messages as per MSC1227 ([\#3589](https://github.com/matrix-org/synapse/issues/3589))
+- Add ability to limit number of monthly active users on the server ([\#3633](https://github.com/matrix-org/synapse/issues/3633))
+- Support more federation endpoints on workers ([\#3653](https://github.com/matrix-org/synapse/issues/3653))
+- Basic support for room versioning ([\#3654](https://github.com/matrix-org/synapse/issues/3654))
+- Ability to disable client/server Synapse via conf toggle ([\#3655](https://github.com/matrix-org/synapse/issues/3655))
+- Ability to whitelist specific threepids against monthly active user limiting ([\#3662](https://github.com/matrix-org/synapse/issues/3662))
+- Add some metrics for the appservice and federation event sending loops ([\#3664](https://github.com/matrix-org/synapse/issues/3664))
+- Where server is disabled, block ability for locked out users to read new messages ([\#3670](https://github.com/matrix-org/synapse/issues/3670))
+- set admin uri via config, to be used in error messages where the user should contact the administrator ([\#3687](https://github.com/matrix-org/synapse/issues/3687))
+- Synapse's presence functionality can now be disabled with the "use_presence" configuration option. ([\#3694](https://github.com/matrix-org/synapse/issues/3694))
+- For resource limit blocked users, prevent writing into rooms ([\#3708](https://github.com/matrix-org/synapse/issues/3708))
+
+
+Bugfixes
+--------
+
+- Fix occasional glitches in the synapse_event_persisted_position metric ([\#3658](https://github.com/matrix-org/synapse/issues/3658))
+- Fix bug on deleting 3pid when using identity servers that don't support unbind API ([\#3661](https://github.com/matrix-org/synapse/issues/3661))
+- Make the tests pass on Twisted < 18.7.0 ([\#3676](https://github.com/matrix-org/synapse/issues/3676))
+- Don’t ship recaptcha_ajax.js, use it directly from Google ([\#3677](https://github.com/matrix-org/synapse/issues/3677))
+- Fixes test_reap_monthly_active_users so it passes under postgres ([\#3681](https://github.com/matrix-org/synapse/issues/3681))
+- Fix mau blocking calulation bug on login ([\#3689](https://github.com/matrix-org/synapse/issues/3689))
+- Fix missing yield in synapse.storage.monthly_active_users.initialise_reserved_users ([\#3692](https://github.com/matrix-org/synapse/issues/3692))
+- Improve HTTP request logging to include all requests ([\#3700](https://github.com/matrix-org/synapse/issues/3700))
+- Avoid timing out requests while we are streaming back the response ([\#3701](https://github.com/matrix-org/synapse/issues/3701))
+- Support more federation endpoints on workers ([\#3705](https://github.com/matrix-org/synapse/issues/3705), [\#3713](https://github.com/matrix-org/synapse/issues/3713))
+- Fix "Starting db txn 'get_all_updated_receipts' from sentinel context" warning ([\#3710](https://github.com/matrix-org/synapse/issues/3710))
+- Fix bug where `state_cache` cache factor ignored environment variables ([\#3719](https://github.com/matrix-org/synapse/issues/3719))
+
+
+Deprecations and Removals
+-------------------------
+
+- The Shared-Secret registration method of the legacy v1/register REST endpoint has been removed. For a replacement, please see [the admin/register API documentation](https://github.com/matrix-org/synapse/blob/master/docs/admin_api/register_api.rst). ([\#3703](https://github.com/matrix-org/synapse/issues/3703))
+
+
+Internal Changes
+----------------
+
+- The test suite now can run under PostgreSQL. ([\#3423](https://github.com/matrix-org/synapse/issues/3423))
+- Refactor HTTP replication endpoints to reduce code duplication ([\#3632](https://github.com/matrix-org/synapse/issues/3632))
+- Tests now correctly execute on Python 3. ([\#3647](https://github.com/matrix-org/synapse/issues/3647))
+- Sytests can now be run inside a Docker container. ([\#3660](https://github.com/matrix-org/synapse/issues/3660))
+- Port over enough to Python 3 to allow the sytests to start. ([\#3668](https://github.com/matrix-org/synapse/issues/3668))
+- Update docker base image from alpine 3.7 to 3.8. ([\#3669](https://github.com/matrix-org/synapse/issues/3669))
+- Rename synapse.util.async to synapse.util.async_helpers to mitigate async becoming a keyword on Python 3.7. ([\#3678](https://github.com/matrix-org/synapse/issues/3678))
+- Synapse's tests are now formatted with the black autoformatter. ([\#3679](https://github.com/matrix-org/synapse/issues/3679))
+- Implemented a new testing base class to reduce test boilerplate. ([\#3684](https://github.com/matrix-org/synapse/issues/3684))
+- Rename MAU prometheus metrics ([\#3690](https://github.com/matrix-org/synapse/issues/3690))
+- add new error type ResourceLimit ([\#3707](https://github.com/matrix-org/synapse/issues/3707))
+- Logcontexts for replication command handlers ([\#3709](https://github.com/matrix-org/synapse/issues/3709))
+- Update admin register API documentation to reference a real user ID. ([\#3712](https://github.com/matrix-org/synapse/issues/3712))
+
+
 Synapse 0.33.2 (2018-08-09)
 ===========================
 
diff --git a/changelog.d/1491.feature b/changelog.d/1491.feature
deleted file mode 100644
index 77b6d6ca09..0000000000
--- a/changelog.d/1491.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add support for the SNI extension to federation TLS connections
\ No newline at end of file
diff --git a/changelog.d/3184.feature b/changelog.d/3184.feature
deleted file mode 100644
index 9f746a57a0..0000000000
--- a/changelog.d/3184.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add /_media/r0/config
diff --git a/changelog.d/3423.misc b/changelog.d/3423.misc
deleted file mode 100644
index 51768c6d14..0000000000
--- a/changelog.d/3423.misc
+++ /dev/null
@@ -1 +0,0 @@
-The test suite now can run under PostgreSQL.
diff --git a/changelog.d/3568.feature b/changelog.d/3568.feature
deleted file mode 100644
index 247f02ba4e..0000000000
--- a/changelog.d/3568.feature
+++ /dev/null
@@ -1 +0,0 @@
-speed up /members API and add `at` and `membership` params as per MSC1227
diff --git a/changelog.d/3574.feature b/changelog.d/3574.feature
deleted file mode 100644
index 87ac32df72..0000000000
--- a/changelog.d/3574.feature
+++ /dev/null
@@ -1 +0,0 @@
-implement `summary` block in /sync response as per MSC688
diff --git a/changelog.d/3589.feature b/changelog.d/3589.feature
deleted file mode 100644
index a8d7124719..0000000000
--- a/changelog.d/3589.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add lazy-loading support to /messages as per MSC1227
diff --git a/changelog.d/3632.misc b/changelog.d/3632.misc
deleted file mode 100644
index 9d64bbe83b..0000000000
--- a/changelog.d/3632.misc
+++ /dev/null
@@ -1 +0,0 @@
-Refactor HTTP replication endpoints to reduce code duplication
diff --git a/changelog.d/3633.feature b/changelog.d/3633.feature
deleted file mode 100644
index 8007a04840..0000000000
--- a/changelog.d/3633.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add ability to limit number of monthly active users on the server
diff --git a/changelog.d/3647.misc b/changelog.d/3647.misc
deleted file mode 100644
index dbc66dae60..0000000000
--- a/changelog.d/3647.misc
+++ /dev/null
@@ -1 +0,0 @@
-Tests now correctly execute on Python 3.
diff --git a/changelog.d/3653.feature b/changelog.d/3653.feature
deleted file mode 100644
index 6c5422994f..0000000000
--- a/changelog.d/3653.feature
+++ /dev/null
@@ -1 +0,0 @@
-Support more federation endpoints on workers
diff --git a/changelog.d/3654.feature b/changelog.d/3654.feature
deleted file mode 100644
index 35c95580bc..0000000000
--- a/changelog.d/3654.feature
+++ /dev/null
@@ -1 +0,0 @@
-Basic support for room versioning
diff --git a/changelog.d/3655.feature b/changelog.d/3655.feature
deleted file mode 100644
index 1134e549e7..0000000000
--- a/changelog.d/3655.feature
+++ /dev/null
@@ -1 +0,0 @@
-Ability to disable client/server Synapse via conf toggle
diff --git a/changelog.d/3658.bugfix b/changelog.d/3658.bugfix
deleted file mode 100644
index 556011a150..0000000000
--- a/changelog.d/3658.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix occasional glitches in the synapse_event_persisted_position metric
diff --git a/changelog.d/3660.misc b/changelog.d/3660.misc
deleted file mode 100644
index acd814c273..0000000000
--- a/changelog.d/3660.misc
+++ /dev/null
@@ -1 +0,0 @@
-Sytests can now be run inside a Docker container.
diff --git a/changelog.d/3661.bugfix b/changelog.d/3661.bugfix
deleted file mode 100644
index f2b4703d80..0000000000
--- a/changelog.d/3661.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug on deleting 3pid when using identity servers that don't support unbind API
diff --git a/changelog.d/3662.feature b/changelog.d/3662.feature
deleted file mode 100644
index daacef086d..0000000000
--- a/changelog.d/3662.feature
+++ /dev/null
@@ -1 +0,0 @@
-Ability to whitelist specific threepids against monthly active user limiting
diff --git a/changelog.d/3664.feature b/changelog.d/3664.feature
deleted file mode 100644
index 184dde9939..0000000000
--- a/changelog.d/3664.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add some metrics for the appservice and federation event sending loops
diff --git a/changelog.d/3668.misc b/changelog.d/3668.misc
deleted file mode 100644
index f9265addf1..0000000000
--- a/changelog.d/3668.misc
+++ /dev/null
@@ -1 +0,0 @@
-Port over enough to Python 3 to allow the sytests to start.
diff --git a/changelog.d/3669.misc b/changelog.d/3669.misc
deleted file mode 100644
index fc579ddc60..0000000000
--- a/changelog.d/3669.misc
+++ /dev/null
@@ -1 +0,0 @@
-Update docker base image from alpine 3.7 to 3.8.
diff --git a/changelog.d/3670.feature b/changelog.d/3670.feature
deleted file mode 100644
index ba00f2d2ec..0000000000
--- a/changelog.d/3670.feature
+++ /dev/null
@@ -1 +0,0 @@
-Where server is disabled, block ability for locked out users to read new messages
diff --git a/changelog.d/3676.bugfix b/changelog.d/3676.bugfix
deleted file mode 100644
index 7b23a2773a..0000000000
--- a/changelog.d/3676.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Make the tests pass on Twisted < 18.7.0
diff --git a/changelog.d/3677.bugfix b/changelog.d/3677.bugfix
deleted file mode 100644
index caa551627b..0000000000
--- a/changelog.d/3677.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Don’t ship recaptcha_ajax.js, use it directly from Google
diff --git a/changelog.d/3678.misc b/changelog.d/3678.misc
deleted file mode 100644
index 0d7c8da64a..0000000000
--- a/changelog.d/3678.misc
+++ /dev/null
@@ -1 +0,0 @@
-Rename synapse.util.async to synapse.util.async_helpers to mitigate async becoming a keyword on Python 3.7.
diff --git a/changelog.d/3679.misc b/changelog.d/3679.misc
deleted file mode 100644
index 1de0a0f2b4..0000000000
--- a/changelog.d/3679.misc
+++ /dev/null
@@ -1 +0,0 @@
-Synapse's tests are now formatted with the black autoformatter.
diff --git a/changelog.d/3681.bugfix b/changelog.d/3681.bugfix
deleted file mode 100644
index d18a69cd0c..0000000000
--- a/changelog.d/3681.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fixes test_reap_monthly_active_users so it passes under postgres
diff --git a/changelog.d/3684.misc b/changelog.d/3684.misc
deleted file mode 100644
index 4c013263c4..0000000000
--- a/changelog.d/3684.misc
+++ /dev/null
@@ -1 +0,0 @@
-Implemented a new testing base class to reduce test boilerplate.
diff --git a/changelog.d/3687.feature b/changelog.d/3687.feature
deleted file mode 100644
index 64b89f6411..0000000000
--- a/changelog.d/3687.feature
+++ /dev/null
@@ -1 +0,0 @@
-set admin uri via config, to be used in error messages where the user should contact the administrator
diff --git a/changelog.d/3689.bugfix b/changelog.d/3689.bugfix
deleted file mode 100644
index 934d039836..0000000000
--- a/changelog.d/3689.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix mau blocking calulation bug on login
diff --git a/changelog.d/3690.misc b/changelog.d/3690.misc
deleted file mode 100644
index 710add0243..0000000000
--- a/changelog.d/3690.misc
+++ /dev/null
@@ -1 +0,0 @@
-Rename MAU prometheus metrics
diff --git a/changelog.d/3692.bugfix b/changelog.d/3692.bugfix
deleted file mode 100644
index f44e13dca1..0000000000
--- a/changelog.d/3692.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix missing yield in synapse.storage.monthly_active_users.initialise_reserved_users
diff --git a/changelog.d/3694.feature b/changelog.d/3694.feature
deleted file mode 100644
index 916a342ff4..0000000000
--- a/changelog.d/3694.feature
+++ /dev/null
@@ -1 +0,0 @@
-Synapse's presence functionality can now be disabled with the "use_presence" configuration option.
diff --git a/changelog.d/3700.bugfix b/changelog.d/3700.bugfix
deleted file mode 100644
index 492cce1dfc..0000000000
--- a/changelog.d/3700.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Improve HTTP request logging to include all requests
\ No newline at end of file
diff --git a/changelog.d/3701.bugfix b/changelog.d/3701.bugfix
deleted file mode 100644
index c22de34537..0000000000
--- a/changelog.d/3701.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Avoid timing out requests while we are streaming back the response
diff --git a/changelog.d/3703.removal b/changelog.d/3703.removal
deleted file mode 100644
index d13ce5adb4..0000000000
--- a/changelog.d/3703.removal
+++ /dev/null
@@ -1 +0,0 @@
-The Shared-Secret registration method of the legacy v1/register REST endpoint has been removed. For a replacement, please see [the admin/register API documentation](https://github.com/matrix-org/synapse/blob/master/docs/admin_api/register_api.rst).
diff --git a/changelog.d/3705.bugfix b/changelog.d/3705.bugfix
deleted file mode 100644
index 6c5422994f..0000000000
--- a/changelog.d/3705.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Support more federation endpoints on workers
diff --git a/changelog.d/3707.misc b/changelog.d/3707.misc
deleted file mode 100644
index 8123ca6543..0000000000
--- a/changelog.d/3707.misc
+++ /dev/null
@@ -1 +0,0 @@
-add new error type ResourceLimit
diff --git a/changelog.d/3708.feature b/changelog.d/3708.feature
deleted file mode 100644
index 2f146ba62b..0000000000
--- a/changelog.d/3708.feature
+++ /dev/null
@@ -1 +0,0 @@
-For resource limit blocked users, prevent writing into rooms
diff --git a/changelog.d/3709.misc b/changelog.d/3709.misc
deleted file mode 100644
index bbda357d44..0000000000
--- a/changelog.d/3709.misc
+++ /dev/null
@@ -1 +0,0 @@
-Logcontexts for replication command handlers
diff --git a/changelog.d/3710.bugfix b/changelog.d/3710.bugfix
deleted file mode 100644
index c28e177667..0000000000
--- a/changelog.d/3710.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix "Starting db txn 'get_all_updated_receipts' from sentinel context" warning
\ No newline at end of file
diff --git a/changelog.d/3712.misc b/changelog.d/3712.misc
deleted file mode 100644
index 30f8c2af21..0000000000
--- a/changelog.d/3712.misc
+++ /dev/null
@@ -1 +0,0 @@
-Update admin register API documentation to reference a real user ID.
diff --git a/changelog.d/3713.bugfix b/changelog.d/3713.bugfix
deleted file mode 100644
index 6c5422994f..0000000000
--- a/changelog.d/3713.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Support more federation endpoints on workers
diff --git a/changelog.d/3719.bugfix b/changelog.d/3719.bugfix
deleted file mode 100644
index 095d9288e8..0000000000
--- a/changelog.d/3719.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug where `state_cache` cache factor ignored environment variables

From 55e6bdf28798153de5c2a6cf9bf0a6618f59168a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 20 Aug 2018 18:20:07 +0100
Subject: [PATCH 4/7] Robustness fix for logcontext filter

Make the logcontext filter not explode if it somehow ends up with a logcontext
of None, since that infinite-loops the whole logging system.
---
 synapse/util/logcontext.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py
index 07e83fadda..848765e530 100644
--- a/synapse/util/logcontext.py
+++ b/synapse/util/logcontext.py
@@ -385,7 +385,13 @@ class LoggingContextFilter(logging.Filter):
         context = LoggingContext.current_context()
         for key, value in self.defaults.items():
             setattr(record, key, value)
-        context.copy_to(record)
+
+        # context should never be None, but if it somehow ends up being, then
+        # we end up in a death spiral of infinite loops, so let's check, for
+        # robustness' sake.
+        if context is not None:
+            context.copy_to(record)
+
         return True
 
 

From be6527325a829e7d5e4540d7a1983a6e6fda2c0f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 20 Aug 2018 18:21:10 +0100
Subject: [PATCH 5/7] Fix exceptions when a connection is closed before we read
 the headers

This fixes bugs introduced in #3700, by making sure that we behave sanely
when an incoming connection is closed before the headers are read.
---
 synapse/http/site.py       | 8 +++++++-
 synapse/util/logcontext.py | 4 +++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/synapse/http/site.py b/synapse/http/site.py
index ad2a98468e..88ed3714f9 100644
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -182,7 +182,7 @@ class SynapseRequest(Request):
         # the client disconnects.
         with PreserveLoggingContext(self.logcontext):
             logger.warn(
-                "Error processing request: %s %s", reason.type, reason.value,
+                "Error processing request %r: %s %s", self, reason.type, reason.value,
             )
 
             if not self._is_processing:
@@ -219,6 +219,12 @@ class SynapseRequest(Request):
         """Log the completion of this request and update the metrics
         """
 
+        if self.logcontext is None:
+            # this can happen if the connection closed before we read the
+            # headers (so render was never called). In that case we'll already
+            # have logged a warning, so just bail out.
+            return
+
         usage = self.logcontext.get_resource_usage()
 
         if self._processing_finished_time is None:
diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py
index 848765e530..a0c2d37610 100644
--- a/synapse/util/logcontext.py
+++ b/synapse/util/logcontext.py
@@ -402,7 +402,9 @@ class PreserveLoggingContext(object):
 
     __slots__ = ["current_context", "new_context", "has_parent"]
 
-    def __init__(self, new_context=LoggingContext.sentinel):
+    def __init__(self, new_context=None):
+        if new_context is None:
+            new_context = LoggingContext.sentinel
         self.new_context = new_context
 
     def __enter__(self):

From 012d612f9d3c8ca58538fffe4207309bf1644cd8 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 20 Aug 2018 18:26:27 +0100
Subject: [PATCH 6/7] changelog

---
 changelog.d/3723.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3723.bugfix

diff --git a/changelog.d/3723.bugfix b/changelog.d/3723.bugfix
new file mode 100644
index 0000000000..7c21083691
--- /dev/null
+++ b/changelog.d/3723.bugfix
@@ -0,0 +1 @@
+Fix bug in v0.33.3rc1 which caused infinite loops and OOMs

From 3b5b64ac99c1a253cc24b6a1063247115ba18cd3 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 21 Aug 2018 03:48:55 +1000
Subject: [PATCH 7/7] changelog

---
 CHANGES.md              | 9 +++++++++
 changelog.d/3723.bugfix | 1 -
 synapse/__init__.py     | 2 +-
 3 files changed, 10 insertions(+), 2 deletions(-)
 delete mode 100644 changelog.d/3723.bugfix

diff --git a/CHANGES.md b/CHANGES.md
index 68ce5d1471..df01178971 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,12 @@
+Synapse 0.33.3rc2 (2018-08-21)
+==============================
+
+Bugfixes
+--------
+
+- Fix bug in v0.33.3rc1 which caused infinite loops and OOMs ([\#3723](https://github.com/matrix-org/synapse/issues/3723))
+
+
 Synapse 0.33.3rc1 (2018-08-21)
 ==============================
 
diff --git a/changelog.d/3723.bugfix b/changelog.d/3723.bugfix
deleted file mode 100644
index 7c21083691..0000000000
--- a/changelog.d/3723.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug in v0.33.3rc1 which caused infinite loops and OOMs
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 5175a1161d..252c49ca82 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -17,4 +17,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.33.3rc1"
+__version__ = "0.33.3rc2"