2015-02-19 21:36:02 +03:00
|
|
|
# -*- coding: utf-8 -*-
|
2016-01-07 07:26:29 +03:00
|
|
|
# Copyright 2015, 2016 OpenMarket Ltd
|
2019-02-20 15:03:30 +03:00
|
|
|
# Copyright 2019 New Vector Ltd
|
2015-02-19 21:36:02 +03:00
|
|
|
#
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
|
# You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
|
|
|
|
|
|
|
|
2016-08-19 17:58:52 +03:00
|
|
|
from mock import Mock
|
|
|
|
|
2018-07-09 09:09:20 +03:00
|
|
|
from twisted.internet import defer
|
2015-08-06 15:33:34 +03:00
|
|
|
|
2018-08-10 16:50:21 +03:00
|
|
|
from synapse.util.async_helpers import ObservableDeferred
|
2015-08-11 19:59:32 +03:00
|
|
|
from synapse.util.caches.descriptors import Cache, cached
|
2015-03-20 21:13:49 +03:00
|
|
|
|
2018-07-09 09:09:20 +03:00
|
|
|
from tests import unittest
|
|
|
|
|
2015-03-20 21:13:49 +03:00
|
|
|
|
|
|
|
class CacheTestCase(unittest.TestCase):
|
|
|
|
def setUp(self):
|
|
|
|
self.cache = Cache("test")
|
|
|
|
|
|
|
|
def test_empty(self):
|
|
|
|
failed = False
|
|
|
|
try:
|
|
|
|
self.cache.get("foo")
|
|
|
|
except KeyError:
|
|
|
|
failed = True
|
|
|
|
|
|
|
|
self.assertTrue(failed)
|
|
|
|
|
|
|
|
def test_hit(self):
|
|
|
|
self.cache.prefill("foo", 123)
|
|
|
|
|
|
|
|
self.assertEquals(self.cache.get("foo"), 123)
|
|
|
|
|
|
|
|
def test_invalidate(self):
|
2015-08-07 13:52:21 +03:00
|
|
|
self.cache.prefill(("foo",), 123)
|
|
|
|
self.cache.invalidate(("foo",))
|
2015-03-20 21:13:49 +03:00
|
|
|
|
|
|
|
failed = False
|
|
|
|
try:
|
2015-08-07 13:52:21 +03:00
|
|
|
self.cache.get(("foo",))
|
2015-03-20 21:13:49 +03:00
|
|
|
except KeyError:
|
|
|
|
failed = True
|
|
|
|
|
|
|
|
self.assertTrue(failed)
|
2015-02-19 21:36:02 +03:00
|
|
|
|
2015-03-25 21:50:43 +03:00
|
|
|
def test_eviction(self):
|
|
|
|
cache = Cache("test", max_entries=2)
|
|
|
|
|
|
|
|
cache.prefill(1, "one")
|
|
|
|
cache.prefill(2, "two")
|
|
|
|
cache.prefill(3, "three") # 1 will be evicted
|
|
|
|
|
|
|
|
failed = False
|
|
|
|
try:
|
|
|
|
cache.get(1)
|
|
|
|
except KeyError:
|
|
|
|
failed = True
|
|
|
|
|
|
|
|
self.assertTrue(failed)
|
|
|
|
|
|
|
|
cache.get(2)
|
|
|
|
cache.get(3)
|
|
|
|
|
2015-03-25 22:05:34 +03:00
|
|
|
def test_eviction_lru(self):
|
2016-08-19 13:59:29 +03:00
|
|
|
cache = Cache("test", max_entries=2)
|
2015-03-25 22:05:34 +03:00
|
|
|
|
|
|
|
cache.prefill(1, "one")
|
|
|
|
cache.prefill(2, "two")
|
|
|
|
|
|
|
|
# Now access 1 again, thus causing 2 to be least-recently used
|
|
|
|
cache.get(1)
|
|
|
|
|
|
|
|
cache.prefill(3, "three")
|
|
|
|
|
|
|
|
failed = False
|
|
|
|
try:
|
|
|
|
cache.get(2)
|
|
|
|
except KeyError:
|
|
|
|
failed = True
|
|
|
|
|
|
|
|
self.assertTrue(failed)
|
|
|
|
|
|
|
|
cache.get(1)
|
|
|
|
cache.get(3)
|
|
|
|
|
2015-02-19 21:36:02 +03:00
|
|
|
|
|
|
|
class CacheDecoratorTestCase(unittest.TestCase):
|
|
|
|
@defer.inlineCallbacks
|
|
|
|
def test_passthrough(self):
|
2015-06-03 16:45:17 +03:00
|
|
|
class A(object):
|
|
|
|
@cached()
|
|
|
|
def func(self, key):
|
|
|
|
return key
|
2015-02-19 21:36:02 +03:00
|
|
|
|
2015-06-03 16:45:17 +03:00
|
|
|
a = A()
|
|
|
|
|
|
|
|
self.assertEquals((yield a.func("foo")), "foo")
|
|
|
|
self.assertEquals((yield a.func("bar")), "bar")
|
2015-02-19 21:36:02 +03:00
|
|
|
|
|
|
|
@defer.inlineCallbacks
|
|
|
|
def test_hit(self):
|
|
|
|
callcount = [0]
|
|
|
|
|
2015-06-03 16:45:17 +03:00
|
|
|
class A(object):
|
|
|
|
@cached()
|
|
|
|
def func(self, key):
|
|
|
|
callcount[0] += 1
|
|
|
|
return key
|
2015-02-19 21:36:02 +03:00
|
|
|
|
2015-06-03 16:45:17 +03:00
|
|
|
a = A()
|
|
|
|
yield a.func("foo")
|
2015-02-19 21:36:02 +03:00
|
|
|
|
|
|
|
self.assertEquals(callcount[0], 1)
|
|
|
|
|
2015-06-03 16:45:17 +03:00
|
|
|
self.assertEquals((yield a.func("foo")), "foo")
|
2015-02-19 21:36:02 +03:00
|
|
|
self.assertEquals(callcount[0], 1)
|
|
|
|
|
|
|
|
@defer.inlineCallbacks
|
|
|
|
def test_invalidate(self):
|
|
|
|
callcount = [0]
|
|
|
|
|
2015-06-03 16:45:17 +03:00
|
|
|
class A(object):
|
|
|
|
@cached()
|
|
|
|
def func(self, key):
|
|
|
|
callcount[0] += 1
|
|
|
|
return key
|
2015-02-19 21:36:02 +03:00
|
|
|
|
2015-06-03 16:45:17 +03:00
|
|
|
a = A()
|
|
|
|
yield a.func("foo")
|
2015-02-19 21:36:02 +03:00
|
|
|
|
|
|
|
self.assertEquals(callcount[0], 1)
|
|
|
|
|
2015-08-07 13:52:21 +03:00
|
|
|
a.func.invalidate(("foo",))
|
2015-02-19 21:36:02 +03:00
|
|
|
|
2015-06-03 16:45:17 +03:00
|
|
|
yield a.func("foo")
|
2015-02-19 21:36:02 +03:00
|
|
|
|
|
|
|
self.assertEquals(callcount[0], 2)
|
|
|
|
|
2015-02-23 21:29:26 +03:00
|
|
|
def test_invalidate_missing(self):
|
2015-06-03 16:45:17 +03:00
|
|
|
class A(object):
|
|
|
|
@cached()
|
|
|
|
def func(self, key):
|
|
|
|
return key
|
2015-02-23 21:29:26 +03:00
|
|
|
|
2015-08-07 13:52:21 +03:00
|
|
|
A().func.invalidate(("what",))
|
2015-02-23 21:29:26 +03:00
|
|
|
|
2015-02-19 21:36:02 +03:00
|
|
|
@defer.inlineCallbacks
|
|
|
|
def test_max_entries(self):
|
|
|
|
callcount = [0]
|
|
|
|
|
2015-06-03 16:45:17 +03:00
|
|
|
class A(object):
|
|
|
|
@cached(max_entries=10)
|
|
|
|
def func(self, key):
|
|
|
|
callcount[0] += 1
|
|
|
|
return key
|
2015-02-19 21:36:02 +03:00
|
|
|
|
2015-06-03 16:45:17 +03:00
|
|
|
a = A()
|
|
|
|
|
|
|
|
for k in range(0, 12):
|
|
|
|
yield a.func(k)
|
2015-02-19 21:36:02 +03:00
|
|
|
|
|
|
|
self.assertEquals(callcount[0], 12)
|
|
|
|
|
|
|
|
# There must have been at least 2 evictions, meaning if we calculate
|
|
|
|
# all 12 values again, we must get called at least 2 more times
|
2016-02-19 18:34:38 +03:00
|
|
|
for k in range(0, 12):
|
2015-06-03 16:45:17 +03:00
|
|
|
yield a.func(k)
|
2015-02-19 21:36:02 +03:00
|
|
|
|
2016-02-19 18:34:38 +03:00
|
|
|
self.assertTrue(
|
2018-08-10 16:54:09 +03:00
|
|
|
callcount[0] >= 14, msg="Expected callcount >= 14, got %d" % (callcount[0])
|
2016-02-19 18:34:38 +03:00
|
|
|
)
|
2015-02-23 18:41:54 +03:00
|
|
|
|
|
|
|
def test_prefill(self):
|
|
|
|
callcount = [0]
|
|
|
|
|
2015-08-06 15:33:34 +03:00
|
|
|
d = defer.succeed(123)
|
|
|
|
|
2015-06-03 16:45:17 +03:00
|
|
|
class A(object):
|
|
|
|
@cached()
|
|
|
|
def func(self, key):
|
|
|
|
callcount[0] += 1
|
2015-08-06 15:33:34 +03:00
|
|
|
return d
|
2015-06-03 16:45:17 +03:00
|
|
|
|
|
|
|
a = A()
|
2015-02-23 18:41:54 +03:00
|
|
|
|
2015-08-07 13:52:21 +03:00
|
|
|
a.func.prefill(("foo",), ObservableDeferred(d))
|
2015-02-23 18:41:54 +03:00
|
|
|
|
2019-10-30 14:35:46 +03:00
|
|
|
self.assertEquals(a.func("foo").result, d.result)
|
2015-02-23 18:41:54 +03:00
|
|
|
self.assertEquals(callcount[0], 0)
|
2016-08-19 13:18:26 +03:00
|
|
|
|
|
|
|
@defer.inlineCallbacks
|
|
|
|
def test_invalidate_context(self):
|
|
|
|
callcount = [0]
|
|
|
|
callcount2 = [0]
|
|
|
|
|
|
|
|
class A(object):
|
|
|
|
@cached()
|
|
|
|
def func(self, key):
|
|
|
|
callcount[0] += 1
|
|
|
|
return key
|
|
|
|
|
2016-08-19 17:02:38 +03:00
|
|
|
@cached(cache_context=True)
|
2016-08-19 13:18:26 +03:00
|
|
|
def func2(self, key, cache_context):
|
|
|
|
callcount2[0] += 1
|
2016-08-19 17:13:58 +03:00
|
|
|
return self.func(key, on_invalidate=cache_context.invalidate)
|
2016-08-19 13:18:26 +03:00
|
|
|
|
|
|
|
a = A()
|
|
|
|
yield a.func2("foo")
|
|
|
|
|
|
|
|
self.assertEquals(callcount[0], 1)
|
|
|
|
self.assertEquals(callcount2[0], 1)
|
|
|
|
|
|
|
|
a.func.invalidate(("foo",))
|
|
|
|
yield a.func("foo")
|
|
|
|
|
|
|
|
self.assertEquals(callcount[0], 2)
|
|
|
|
self.assertEquals(callcount2[0], 1)
|
|
|
|
|
|
|
|
yield a.func2("foo")
|
|
|
|
|
|
|
|
self.assertEquals(callcount[0], 2)
|
|
|
|
self.assertEquals(callcount2[0], 2)
|
|
|
|
|
|
|
|
@defer.inlineCallbacks
|
|
|
|
def test_eviction_context(self):
|
|
|
|
callcount = [0]
|
|
|
|
callcount2 = [0]
|
|
|
|
|
|
|
|
class A(object):
|
2017-07-04 12:21:25 +03:00
|
|
|
@cached(max_entries=4) # HACK: This makes it 2 due to cache factor
|
2016-08-19 13:18:26 +03:00
|
|
|
def func(self, key):
|
|
|
|
callcount[0] += 1
|
|
|
|
return key
|
|
|
|
|
2016-08-19 17:02:38 +03:00
|
|
|
@cached(cache_context=True)
|
2016-08-19 13:18:26 +03:00
|
|
|
def func2(self, key, cache_context):
|
|
|
|
callcount2[0] += 1
|
2016-08-19 17:13:58 +03:00
|
|
|
return self.func(key, on_invalidate=cache_context.invalidate)
|
2016-08-19 13:18:26 +03:00
|
|
|
|
|
|
|
a = A()
|
|
|
|
yield a.func2("foo")
|
|
|
|
yield a.func2("foo2")
|
|
|
|
|
|
|
|
self.assertEquals(callcount[0], 2)
|
|
|
|
self.assertEquals(callcount2[0], 2)
|
|
|
|
|
2017-01-17 14:18:13 +03:00
|
|
|
yield a.func2("foo")
|
|
|
|
self.assertEquals(callcount[0], 2)
|
|
|
|
self.assertEquals(callcount2[0], 2)
|
|
|
|
|
2016-08-19 13:18:26 +03:00
|
|
|
yield a.func("foo3")
|
|
|
|
|
|
|
|
self.assertEquals(callcount[0], 3)
|
|
|
|
self.assertEquals(callcount2[0], 2)
|
|
|
|
|
|
|
|
yield a.func2("foo")
|
|
|
|
|
|
|
|
self.assertEquals(callcount[0], 4)
|
|
|
|
self.assertEquals(callcount2[0], 3)
|
2016-08-19 17:58:52 +03:00
|
|
|
|
|
|
|
@defer.inlineCallbacks
|
|
|
|
def test_double_get(self):
|
|
|
|
callcount = [0]
|
|
|
|
callcount2 = [0]
|
|
|
|
|
|
|
|
class A(object):
|
|
|
|
@cached()
|
|
|
|
def func(self, key):
|
|
|
|
callcount[0] += 1
|
|
|
|
return key
|
|
|
|
|
|
|
|
@cached(cache_context=True)
|
|
|
|
def func2(self, key, cache_context):
|
|
|
|
callcount2[0] += 1
|
|
|
|
return self.func(key, on_invalidate=cache_context.invalidate)
|
|
|
|
|
|
|
|
a = A()
|
|
|
|
a.func2.cache.cache = Mock(wraps=a.func2.cache.cache)
|
|
|
|
|
|
|
|
yield a.func2("foo")
|
|
|
|
|
|
|
|
self.assertEquals(callcount[0], 1)
|
|
|
|
self.assertEquals(callcount2[0], 1)
|
|
|
|
|
|
|
|
a.func2.invalidate(("foo",))
|
|
|
|
self.assertEquals(a.func2.cache.cache.pop.call_count, 1)
|
|
|
|
|
|
|
|
yield a.func2("foo")
|
|
|
|
a.func2.invalidate(("foo",))
|
|
|
|
self.assertEquals(a.func2.cache.cache.pop.call_count, 2)
|
|
|
|
|
|
|
|
self.assertEquals(callcount[0], 1)
|
|
|
|
self.assertEquals(callcount2[0], 2)
|
|
|
|
|
|
|
|
a.func.invalidate(("foo",))
|
|
|
|
self.assertEquals(a.func2.cache.cache.pop.call_count, 3)
|
|
|
|
yield a.func("foo")
|
|
|
|
|
|
|
|
self.assertEquals(callcount[0], 2)
|
|
|
|
self.assertEquals(callcount2[0], 2)
|
|
|
|
|
|
|
|
yield a.func2("foo")
|
|
|
|
|
|
|
|
self.assertEquals(callcount[0], 2)
|
|
|
|
self.assertEquals(callcount2[0], 3)
|
2019-02-20 15:03:30 +03:00
|
|
|
|
|
|
|
|
|
|
|
class UpsertManyTests(unittest.HomeserverTestCase):
|
|
|
|
def prepare(self, reactor, clock, hs):
|
|
|
|
self.storage = hs.get_datastore()
|
|
|
|
|
|
|
|
self.table_name = "table_" + hs.get_secrets().token_hex(6)
|
|
|
|
self.get_success(
|
2019-12-04 16:52:46 +03:00
|
|
|
self.storage.db.runInteraction(
|
2019-02-20 15:03:30 +03:00
|
|
|
"create",
|
|
|
|
lambda x, *a: x.execute(*a),
|
|
|
|
"CREATE TABLE %s (id INTEGER, username TEXT, value TEXT)"
|
|
|
|
% (self.table_name,),
|
|
|
|
)
|
|
|
|
)
|
|
|
|
self.get_success(
|
2019-12-04 16:52:46 +03:00
|
|
|
self.storage.db.runInteraction(
|
2019-02-20 15:03:30 +03:00
|
|
|
"index",
|
|
|
|
lambda x, *a: x.execute(*a),
|
|
|
|
"CREATE UNIQUE INDEX %sindex ON %s(id, username)"
|
|
|
|
% (self.table_name, self.table_name),
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
def _dump_to_tuple(self, res):
|
|
|
|
for i in res:
|
|
|
|
yield (i["id"], i["username"], i["value"])
|
|
|
|
|
|
|
|
def test_upsert_many(self):
|
|
|
|
"""
|
|
|
|
Upsert_many will perform the upsert operation across a batch of data.
|
|
|
|
"""
|
|
|
|
# Add some data to an empty table
|
|
|
|
key_names = ["id", "username"]
|
|
|
|
value_names = ["value"]
|
|
|
|
key_values = [[1, "user1"], [2, "user2"]]
|
|
|
|
value_values = [["hello"], ["there"]]
|
|
|
|
|
|
|
|
self.get_success(
|
2019-12-04 16:52:46 +03:00
|
|
|
self.storage.db.runInteraction(
|
2019-02-20 15:03:30 +03:00
|
|
|
"test",
|
2019-12-04 16:52:46 +03:00
|
|
|
self.storage.db.simple_upsert_many_txn,
|
2019-02-20 15:03:30 +03:00
|
|
|
self.table_name,
|
|
|
|
key_names,
|
|
|
|
key_values,
|
|
|
|
value_names,
|
|
|
|
value_values,
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
# Check results are what we expect
|
|
|
|
res = self.get_success(
|
2019-12-04 16:52:46 +03:00
|
|
|
self.storage.db.simple_select_list(
|
2019-02-20 15:03:30 +03:00
|
|
|
self.table_name, None, ["id, username, value"]
|
|
|
|
)
|
|
|
|
)
|
|
|
|
self.assertEqual(
|
|
|
|
set(self._dump_to_tuple(res)),
|
2020-02-21 15:15:07 +03:00
|
|
|
{(1, "user1", "hello"), (2, "user2", "there")},
|
2019-02-20 15:03:30 +03:00
|
|
|
)
|
|
|
|
|
|
|
|
# Update only user2
|
|
|
|
key_values = [[2, "user2"]]
|
|
|
|
value_values = [["bleb"]]
|
|
|
|
|
|
|
|
self.get_success(
|
2019-12-04 16:52:46 +03:00
|
|
|
self.storage.db.runInteraction(
|
2019-02-20 15:03:30 +03:00
|
|
|
"test",
|
2019-12-04 16:52:46 +03:00
|
|
|
self.storage.db.simple_upsert_many_txn,
|
2019-02-20 15:03:30 +03:00
|
|
|
self.table_name,
|
|
|
|
key_names,
|
|
|
|
key_values,
|
|
|
|
value_names,
|
|
|
|
value_values,
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
# Check results are what we expect
|
|
|
|
res = self.get_success(
|
2019-12-04 16:52:46 +03:00
|
|
|
self.storage.db.simple_select_list(
|
2019-02-20 15:03:30 +03:00
|
|
|
self.table_name, None, ["id, username, value"]
|
|
|
|
)
|
|
|
|
)
|
|
|
|
self.assertEqual(
|
|
|
|
set(self._dump_to_tuple(res)),
|
2020-02-21 15:15:07 +03:00
|
|
|
{(1, "user1", "hello"), (2, "user2", "bleb")},
|
2019-02-20 15:03:30 +03:00
|
|
|
)
|