diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py
index 332ba4b128..d61061434c 100644
--- a/synapse/rest/media/v1/media_repository.py
+++ b/synapse/rest/media/v1/media_repository.py
@@ -386,9 +386,6 @@ class MediaRepository(object):
             except NotRetryingDestination:
                 logger.warn("Not retrying destination %r", server_name)
                 raise SynapseError(502, "Failed to fetch remote media")
-            except FederationDeniedError as e:
-                logger.info(e)
-                raise e
             except Exception:
                 logger.exception("Failed to fetch remote media %s/%s",
                                  server_name, media_id)