diff --git a/synapse/storage/data_stores/main/deviceinbox.py b/synapse/storage/data_stores/main/deviceinbox.py
index 6393c4f6de..f04aad0743 100644
--- a/synapse/storage/data_stores/main/deviceinbox.py
+++ b/synapse/storage/data_stores/main/deviceinbox.py
@@ -358,14 +358,7 @@ class DeviceInboxStore(DeviceInboxWorkerStore, DeviceInboxBackgroundUpdateStore)
     def _add_messages_to_local_device_inbox_txn(
         self, txn, stream_id, messages_by_user_then_device
     ):
-        # Compatible method of performing an upsert
-        sql = """
-            INSERT INTO device_max_stream_id
-            (stream_id) VALUES (?)
-            ON CONFLICT DO UPDATE device_max_stream_id
-            SET stream_id = ?
-            WHERE stream_id < ?
-        """
+        sql = "UPDATE device_max_stream_id" " SET stream_id = ?" " WHERE stream_id < ?"
         txn.execute(sql, (stream_id, stream_id))
 
         local_by_user_then_device = {}
diff --git a/synapse/storage/data_stores/main/schema/delta/56/device_stream_id.sql b/synapse/storage/data_stores/main/schema/delta/56/device_stream_id.sql
new file mode 100644
index 0000000000..ae261a70a2
--- /dev/null
+++ b/synapse/storage/data_stores/main/schema/delta/56/device_stream_id.sql
@@ -0,0 +1,5 @@
+INSERT INTO device_max_stream_id (stream_id)
+SELECT 0
+WHERE NOT EXISTS (
+    SELECT 1 FROM device_max_stream_id
+);
\ No newline at end of file