chore: release v1.6.1

CHANGELOG.md

 # Changelog
 
+### 1.6.1 (2021-07-28)
+* **backend:** feat: lower redis usage by adding etag caching for large values
+
 ### 1.6.0 (2021-07-27)
 * **backend:** fix: schema validation mismatch
 * **backend:** feat: added dev setup for signing tool

services/backend/package.json

@@ -26,6 +26,7 @@
     "@lucaapp/crypto": "2.0.3",
     "@tsconfig/node14": "1.0.1",
     "@types/config": "0.0.39",
+    "@types/etag": "1.8.1",
     "@types/express": "4.17.13",
     "@types/validator": "13.6.3",
     "axios": "0.21.1",

services/backend/src/routes/v3/badges.js

@@ -14,10 +14,7 @@ const {
 const { validateSchema } = require('../../middlewares/validateSchema');
 const { limitRequestsPerHour } = require('../../middlewares/rateLimit');
 
-const {
-  getBloomFilter,
-  getBloomFilterEtag,
-} = require('../../utils/bloomFilter');
+const { getBloomFilterAndEtag } = require('../../utils/bloomFilter');
 
 const database = require('../../database');
 const { badgeCreateSchema } = require('./badges.schemas');
@@ -78,12 +75,11 @@ router.post(
 );
 
 router.get('/bloomFilter', async (request, response) => {
-  const bloomFilterEtag = await getBloomFilterEtag();
+  const [bloomFilter, bloomFilterEtag] = await getBloomFilterAndEtag();
   if (bloomFilterEtag === request.headers['If-None-Match']) {
     return response.sendStatus(status.NOT_MODIFIED);
   }
 
-  const bloomFilter = await getBloomFilter();
   if (!bloomFilter) {
     return response.sendStatus(status.NOT_FOUND);
   }

services/backend/src/utils/bloomFilter.js

 const moment = require('moment');
-const etag = require('etag');
 const { Worker } = require('worker_threads');
 const { Op } = require('sequelize');
 
@@ -10,9 +9,9 @@ const {
   set: redisSet,
   get: redisGet,
 } = require('./redis');
+const cache = require('./redisCache');
 const database = require('../database');
 
-const BLOOM_FILTER_ETAG_KEY = 'BadgeBloomFilterEtag';
 const BLOOM_FILTER_BUFFER_KEY = 'BadgeBloomFilterBuffer';
 const BLOOM_FILTER_STATE_GENERATING_KEY = 'IsBloomFilterGenerating';
 const BLOOM_FILTER_STATE_EMPTY_BADGE_KEY = 'LastEmptyBadgeCount';
@@ -123,19 +122,15 @@ const updateBloomFilter = async () => {
   worker.postMessage(await getUnregisteredBadges());
 };
 
-const getBloomFilter = () =>
-  redisGet(Buffer.from(BLOOM_FILTER_BUFFER_KEY)).catch(() => null);
-
-const getBloomFilterEtag = () => redisGet(BLOOM_FILTER_ETAG_KEY);
+const getBloomFilterAndEtag = () => cache.get(BLOOM_FILTER_BUFFER_KEY, true);
 
 worker.on('message', async bloomFilterArrayDump => {
   const emptyBadgeCount = await getEmptyBadgeCount();
   const totalBadgeCount = await getTotalBadgeCount();
   const bloomFilterBuffer = Buffer.from(bloomFilterArrayDump);
+  cache.set(BLOOM_FILTER_BUFFER_KEY, bloomFilterBuffer);
   redisClient
     .multi()
-    .set(BLOOM_FILTER_BUFFER_KEY, bloomFilterBuffer)
-    .set(BLOOM_FILTER_ETAG_KEY, etag(bloomFilterBuffer))
     .set(BLOOM_FILTER_STATE_GENERATING_KEY, false)
     .set(BLOOM_FILTER_STATE_EMPTY_BADGE_KEY, (emptyBadgeCount || 0).toString())
     .set(BLOOM_FILTER_STATE_TOTAL_BADGE_KEY, (totalBadgeCount || 0).toString())
@@ -146,6 +141,5 @@ worker.on('message', async bloomFilterArrayDump => {
 
 module.exports = {
   updateBloomFilter,
-  getBloomFilter,
-  getBloomFilterEtag,
+  getBloomFilterAndEtag,
 };

services/backend/src/utils/notifications.js

@@ -10,10 +10,10 @@ const {
   HMAC_SHA256,
 } = require('@lucaapp/crypto');
 
-const redis = require('./redis');
+const cache = require('./redisCache');
 const database = require('../database');
 
-const NOTIFICATIONS_CACHE_KEY = 'cache:notifications';
+const NOTIFICATIONS_CACHE_KEY = 'notifications';
 const {
   DEVICE_TYPE_IOS,
   DEVICE_TYPE_ANDROID,
@@ -103,9 +103,10 @@ const generateNotifications = async () => {
     }))
     .filter(healthDepartment => healthDepartment.hashedTraceIds.length > 0);
 
-  redis.set(NOTIFICATIONS_CACHE_KEY, JSON.stringify(responseValue));
+  cache.set(NOTIFICATIONS_CACHE_KEY, JSON.stringify(responseValue));
 };
 
-const getNotifications = () => redis.get(NOTIFICATIONS_CACHE_KEY);
+const getNotifications = () =>
+  cache.get(NOTIFICATIONS_CACHE_KEY).then(cachedResponse => cachedResponse[0]);
 
 module.exports = { generateNotifications, getNotifications };

services/backend/src/utils/redisCache.ts

+import etag from 'etag';
+import { set as redisSet, get as redisGet } from './redis';
+
+const getDataKey = (key: string) => `cache:data:${key}`;
+const getEtagKey = (key: string) => `cache:etag:${key}`;
+
+const cacheEtagStore = new Map();
+const cacheDataStore = new Map();
+
+const get = async (key: string, bufferStore = false) => {
+  const localEtag = cacheEtagStore.get(key);
+  const remoteEtag = await redisGet(getEtagKey(key));
+  if (localEtag === remoteEtag) {
+    return [cacheDataStore.get(key), localEtag];
+  }
+
+  // If the key is passed as buffer the response will also be a buffer.
+  const dataKey = bufferStore ? Buffer.from(getDataKey(key)) : getDataKey(key);
+  const remoteValue = await redisGet(dataKey);
+
+  cacheDataStore.set(key, remoteValue);
+  cacheEtagStore.set(key, remoteEtag);
+  return [remoteValue, remoteEtag];
+};
+
+const set = (key: string, value: string | Buffer) => {
+  const calculatedEtag = etag(value);
+  const dataKey = getDataKey(key);
+
+  redisSet(dataKey, value);
+  redisSet(getEtagKey(key), calculatedEtag);
+  cacheEtagStore.set(key, calculatedEtag);
+  cacheDataStore.set(key, value);
+};
+
+const cache = { get, set };
+
+module.exports = cache;

services/backend/yarn.lock

@@ -416,6 +416,13 @@
   resolved "https://registry.yarnpkg.com/@types/cookiejar/-/cookiejar-2.1.2.tgz#66ad9331f63fe8a3d3d9d8c6e3906dd10f6446e8"
   integrity sha512-t73xJJrvdTjXrn4jLS9VSGRbz0nUY3cl2DMGDU48lKl+HR9dbbjW2A9r3g40VA++mQpy6uuHg33gy7du2BKpog==
 
+"@types/etag@1.8.1":
+  version "1.8.1"
+  resolved "https://registry.yarnpkg.com/@types/etag/-/etag-1.8.1.tgz#593ca8ddb43acb3db049bd0955fd64d281ab58b9"
+  integrity sha512-bsKkeSqN7HYyYntFRAmzcwx/dKW4Wa+KVMTInANlI72PWLQmOpZu96j0OqHZGArW4VQwCmJPteQlXaUDeOB0WQ==
+  dependencies:
+    "@types/node" "*"
+
 "@types/expect@24.3.0":
   version "24.3.0"
   resolved "https://registry.yarnpkg.com/@types/expect/-/expect-24.3.0.tgz#d7cab8b3c10c2d92a0cbb31981feceb81d3486f1"