diff --git a/src/components/views/settings/CrossSigningPanel.js b/src/components/views/settings/CrossSigningPanel.js
index 669c2e84d9..1c548bd9d8 100644
--- a/src/components/views/settings/CrossSigningPanel.js
+++ b/src/components/views/settings/CrossSigningPanel.js
@@ -187,12 +187,18 @@ export default class CrossSigningPanel extends React.PureComponent {
         }
 
         const keysExistAnywhere = (
+            crossSigningPublicKeysOnDevice ||
             crossSigningPrivateKeysInStorage ||
-            crossSigningPublicKeysOnDevice
+            masterPrivateKeyCached ||
+            selfSigningPrivateKeyCached ||
+            userSigningPrivateKeyCached
         );
         const keysExistEverywhere = (
+            crossSigningPublicKeysOnDevice &&
             crossSigningPrivateKeysInStorage &&
-            crossSigningPublicKeysOnDevice
+            masterPrivateKeyCached &&
+            selfSigningPrivateKeyCached &&
+            userSigningPrivateKeyCached
         );
 
         const actions = [];