feat: warm caches during app startup

This code runs before the public app begins accepting requests, which should mean that we don't need to hit the database very much after beginning to serve requests. Doing this at start-up also means we fetch the releases sequentially, which should (slightly) spread the load out more.

I verified locally that the app is incapable of responding to requests until `public.py` finishes executing by adding a `time.sleep` to the end of it. While the sleep was running requests to the app hung. This means that `/__heartbeat__` requests won't complete successfully until caches have been warmed up.

Hopefully this will help with database load during scaling events. See: https://mozilla-hub.atlassian.net/browse/SVCSE-3425

Author: bhearsum

src/auslib/web/public/helpers.py

@@ -5,6 +5,7 @@
 
 from auslib.AUS import AUS
 from auslib.global_state import cache, dbo
+from auslib.services.releases import get_release_blob
 from auslib.util.autograph import make_hash, sign_hash
 
 log = logging.getLogger(__name__)
@@ -52,4 +53,24 @@ def sign():
     return headers
 
 
+def warm_caches():
+    """Fetch all releases pointed at by a rule. This is intended to run at app
+    startup, to ensure that it has all necessary information is in memory before
+    it begins serving requests."""
+    mapped_releases = set()
+    for rule in dbo.rules.select(columns=[dbo.rules.mapping, dbo.rules.fallbackMapping], distinct=True):
+        if rule["mapping"]:
+            mapped_releases.add(rule["mapping"])
+        if rule["fallbackMapping"]:
+            mapped_releases.add(rule["fallbackMapping"])
+
+    # TODO: fetch all mappings, fallbackMappings, and blobs referenced in a SuperBlob
+    for release in mapped_releases:
+        blob = get_release_blob(release)
+        if blob and blob["schema_version"] == 4000:
+            if response_blobs := blob.getResponseBlobs():
+                for response_blob in response_blobs:
+                    get_release_blob(response_blob)
+
+
 AUS = AUS()

tests/web/test_helpers.py

@@ -1,7 +1,8 @@
 from unittest.mock import MagicMock
 
 import auslib.web.public.helpers
-import auslib.web.public.json
+from auslib.blobs.base import createBlob
+from auslib.global_state import cache, dbo
 
 
 def test_get_content_signature_headers(monkeypatch):
@@ -32,3 +33,74 @@ def mock_sign_hash(_, key, *__):
     assert auslib.web.public.helpers.get_content_signature_headers(content, product) == {"Content-Signature": f"x5u={x5u}; p384ecdsa={ecdsa}"}
 
     assert mocksign.call_count == 1
+
+
+def test_warm_caches(db_schema, insert_release, firefox_54_0_1_build1, firefox_56_0_build1, superblob_e8f4a19, hotfix_bug_1548973_1_1_4, timecop_1_0):
+    dbo.setDb("sqlite:///:memory:")
+    db_schema.create_all(dbo.engine)
+    dbo.rules.t.insert().execute(
+        rule_id=5,
+        priority=90,
+        backgroundRate=100,
+        mapping="Firefox-56.0-build1",
+        fallbackMapping="Firefox-54.0.1-build1",
+        update_type="minor",
+        product="Firefox",
+        mig64=True,
+        data_version=1,
+    )
+    dbo.rules.t.insert().execute(
+        priority=300,
+        product="SystemAddons",
+        channel="releasesjson",
+        mapping="Superblob-e8f4a19cfd695bf0eb66a2115313c31cc23a2369c0dc7b736d2f66d9075d7c66",
+        backgroundRate=100,
+        update_type="minor",
+        data_version=1,
+    )
+    dbo.releases.t.insert().execute(
+        name="Firefox-54.0.1-build1",
+        product="Firefox",
+        data_version=1,
+        data=createBlob(firefox_54_0_1_build1),
+    )
+    insert_release(firefox_56_0_build1, "Firefox", history=False)
+    insert_release(superblob_e8f4a19, "SystemAddons", history=False)
+    insert_release(hotfix_bug_1548973_1_1_4, "SystemAddons", history=False)
+    insert_release(timecop_1_0, "SystemAddons", history=False)
+    cache.reset()
+    cache.make_cache("blob", 50, 3600)
+    cache.make_cache("releases", 50, 3600)
+    cache.make_cache("release_assets", 50, 3600)
+
+    for cache_name in ("blob", "releases", "release_assets"):
+        c = cache.caches[cache_name]
+        assert c.lookups == 0, cache_name
+        assert c.hits == 0, cache_name
+        assert c.misses == 0, cache_name
+
+    auslib.web.public.helpers.warm_caches()
+
+    # one lookup per release, all misses
+    assert cache.caches["blob"].lookups == 1
+    assert cache.caches["blob"].hits == 0
+    assert cache.caches["blob"].misses == 1
+    for cache_name in ("releases", "release_assets"):
+        c = cache.caches[cache_name]
+        # we account for the v1 release here as well, because these tables are
+        # checked before the v1 release table
+        assert c.lookups == 5, cache_name
+        assert c.hits == 0, cache_name
+        assert c.misses == 5, cache_name
+
+    auslib.web.public.helpers.warm_caches()
+
+    # another lookup per release, all hits this time
+    assert cache.caches["blob"].lookups == 2
+    assert cache.caches["blob"].hits == 1
+    assert cache.caches["blob"].misses == 1
+    for cache_name in ("releases", "release_assets"):
+        c = cache.caches[cache_name]
+        assert c.lookups == 10, cache_name
+        assert c.hits == 5, cache_name
+        assert c.misses == 5, cache_name

uvicorn/public.py

@@ -133,3 +133,7 @@
 
 if STAGING:
     application.config["SWAGGER_DEBUG"] = True
+
+from auslib.web.public.helpers import warm_caches
+
+warm_caches()