mirrors/synapse
1
0
Fork 0
mirror of https://github.com/element-hq/synapse.git synced 2024-11-25 19:15:51 +03:00
Code Issues Projects Releases Packages Wiki Activity

Add some metrics

Browse source
This commit is contained in:
Erik Johnston 2016-11-21 17:34:43 +00:00
parent 50934ce460
commit 88d85ebae1
1 changed files with 61 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

102
synapse/federation/send_queue.py
View file

@ -31,10 +31,16 @@ Events are replicated via a separate events stream.
from .units import Edu from .units import Edu
from synapse.util.metrics import Measure
import synapse.metrics
from blist import sorteddict from blist import sorteddict
import ujson import ujson
metrics = synapse.metrics.get_metrics_for(__name__)
PRESENCE_TYPE = "p" PRESENCE_TYPE = "p"
KEYED_EDU_TYPE = "k" KEYED_EDU_TYPE = "k"
EDU_TYPE = "e" EDU_TYPE = "e"
@ -49,8 +55,6 @@ class FederationRemoteSendQueue(object):
self.server_name = hs.hostname self.server_name = hs.hostname
self.clock = hs.get_clock() self.clock = hs.get_clock()
# TODO: Add metrics for size of lists below
self.presence_map = {} self.presence_map = {}
self.presence_changed = sorteddict() self.presence_changed = sorteddict()
@ -61,10 +65,24 @@ class FederationRemoteSendQueue(object):
self.failures = sorteddict() self.failures = sorteddict()
self.device_messages = sorteddict()
self.pos = 1 self.pos = 1
self.pos_time = sorteddict() self.pos_time = sorteddict()
self.device_messages = sorteddict() # EVERYTHING IS SAD. In particular, python only makes new scopes when
# we make a new function, so we need to make a new function so the inner
def register(name, queue):
metrics.register_callback(
queue_name + "_size",
lambda: len(queue),
)
for queue_name in [
"presence_map", "presence_changed", "keyed_edu", "keyed_edu_changed",
"edus", "failures", "device_messages", "pos_time",
]:
register(queue_name, getattr(self, queue_name))
self.clock.looping_call(self._clear_queue, 30 * 1000) self.clock.looping_call(self._clear_queue, 30 * 1000)
@ -76,7 +94,6 @@ class FederationRemoteSendQueue(object):
def _clear_queue(self): def _clear_queue(self):
"""Clear the queues for anything older than N minutes""" """Clear the queues for anything older than N minutes"""
# TODO measure this function time.
FIVE_MINUTES_AGO = 5 * 60 * 1000 FIVE_MINUTES_AGO = 5 * 60 * 1000
now = self.clock.time_msec() now = self.clock.time_msec()
@ -94,51 +111,54 @@ class FederationRemoteSendQueue(object):
def _clear_queue_before_pos(self, position_to_delete): def _clear_queue_before_pos(self, position_to_delete):
"""Clear all the queues from before a given position""" """Clear all the queues from before a given position"""
# Delete things out of presence maps with Measure(self.clock, "send_queue._clear"):
keys = self.presence_changed.keys() # Delete things out of presence maps
i = keys.bisect_left(position_to_delete) keys = self.presence_changed.keys()
for key in keys[:i]: i = keys.bisect_left(position_to_delete)
del self.presence_changed[key] for key in keys[:i]:
del self.presence_changed[key]
user_ids = set( user_ids = set(
user_id for uids in self.presence_changed.values() for _, user_id in uids user_id for uids in self.presence_changed.values() for _, user_id in uids
) )
to_del = [user_id for user_id in self.presence_map if user_id not in user_ids] to_del = [
for user_id in to_del: user_id for user_id in self.presence_map if user_id not in user_ids
del self.presence_map[user_id] ]
for user_id in to_del:
del self.presence_map[user_id]
# Delete things out of keyed edus # Delete things out of keyed edus
keys = self.keyed_edu_changed.keys() keys = self.keyed_edu_changed.keys()
i = keys.bisect_left(position_to_delete) i = keys.bisect_left(position_to_delete)
for key in keys[:i]: for key in keys[:i]:
del self.keyed_edu_changed[key] del self.keyed_edu_changed[key]
live_keys = set() live_keys = set()
for edu_key in self.keyed_edu_changed.values(): for edu_key in self.keyed_edu_changed.values():
live_keys.add(edu_key) live_keys.add(edu_key)
to_del = [edu_key for edu_key in self.keyed_edu if edu_key not in live_keys] to_del = [edu_key for edu_key in self.keyed_edu if edu_key not in live_keys]
for edu_key in to_del: for edu_key in to_del:
del self.keyed_edu[edu_key] del self.keyed_edu[edu_key]
# Delete things out of edu map # Delete things out of edu map
keys = self.edus.keys() keys = self.edus.keys()
i = keys.bisect_left(position_to_delete) i = keys.bisect_left(position_to_delete)
for key in keys[:i]: for key in keys[:i]:
del self.edus[key] del self.edus[key]
# Delete things out of failure map # Delete things out of failure map
keys = self.failures.keys() keys = self.failures.keys()
i = keys.bisect_left(position_to_delete) i = keys.bisect_left(position_to_delete)
for key in keys[:i]: for key in keys[:i]:
del self.failures[key] del self.failures[key]
# Delete things out of device map # Delete things out of device map
keys = self.device_messages.keys() keys = self.device_messages.keys()
i = keys.bisect_left(position_to_delete) i = keys.bisect_left(position_to_delete)
for key in keys[:i]: for key in keys[:i]:
del self.device_messages[key] del self.device_messages[key]
def notify_new_events(self, current_id): def notify_new_events(self, current_id):
"""As per TransactionQueue""" """As per TransactionQueue"""