Unverified Commit c1d9883b authored by Philipp Berger's avatar Philipp Berger
Browse files

chore: release v1.6.1

parent 21fdea04
# Changelog
### 1.6.1 (2021-07-28)
* **backend:** feat: lower redis usage by adding etag caching for large values
### 1.6.0 (2021-07-27)
* **backend:** fix: schema validation mismatch
* **backend:** feat: added dev setup for signing tool
......
......@@ -26,6 +26,7 @@
"@lucaapp/crypto": "2.0.3",
"@tsconfig/node14": "1.0.1",
"@types/config": "0.0.39",
"@types/etag": "1.8.1",
"@types/express": "4.17.13",
"@types/validator": "13.6.3",
"axios": "0.21.1",
......
......@@ -14,10 +14,7 @@ const {
const { validateSchema } = require('../../middlewares/validateSchema');
const { limitRequestsPerHour } = require('../../middlewares/rateLimit');
const {
getBloomFilter,
getBloomFilterEtag,
} = require('../../utils/bloomFilter');
const { getBloomFilterAndEtag } = require('../../utils/bloomFilter');
const database = require('../../database');
const { badgeCreateSchema } = require('./badges.schemas');
......@@ -78,12 +75,11 @@ router.post(
);
router.get('/bloomFilter', async (request, response) => {
const bloomFilterEtag = await getBloomFilterEtag();
const [bloomFilter, bloomFilterEtag] = await getBloomFilterAndEtag();
if (bloomFilterEtag === request.headers['If-None-Match']) {
return response.sendStatus(status.NOT_MODIFIED);
}
const bloomFilter = await getBloomFilter();
if (!bloomFilter) {
return response.sendStatus(status.NOT_FOUND);
}
......
const moment = require('moment');
const etag = require('etag');
const { Worker } = require('worker_threads');
const { Op } = require('sequelize');
......@@ -10,9 +9,9 @@ const {
set: redisSet,
get: redisGet,
} = require('./redis');
const cache = require('./redisCache');
const database = require('../database');
const BLOOM_FILTER_ETAG_KEY = 'BadgeBloomFilterEtag';
const BLOOM_FILTER_BUFFER_KEY = 'BadgeBloomFilterBuffer';
const BLOOM_FILTER_STATE_GENERATING_KEY = 'IsBloomFilterGenerating';
const BLOOM_FILTER_STATE_EMPTY_BADGE_KEY = 'LastEmptyBadgeCount';
......@@ -123,19 +122,15 @@ const updateBloomFilter = async () => {
worker.postMessage(await getUnregisteredBadges());
};
const getBloomFilter = () =>
redisGet(Buffer.from(BLOOM_FILTER_BUFFER_KEY)).catch(() => null);
const getBloomFilterEtag = () => redisGet(BLOOM_FILTER_ETAG_KEY);
const getBloomFilterAndEtag = () => cache.get(BLOOM_FILTER_BUFFER_KEY, true);
worker.on('message', async bloomFilterArrayDump => {
const emptyBadgeCount = await getEmptyBadgeCount();
const totalBadgeCount = await getTotalBadgeCount();
const bloomFilterBuffer = Buffer.from(bloomFilterArrayDump);
cache.set(BLOOM_FILTER_BUFFER_KEY, bloomFilterBuffer);
redisClient
.multi()
.set(BLOOM_FILTER_BUFFER_KEY, bloomFilterBuffer)
.set(BLOOM_FILTER_ETAG_KEY, etag(bloomFilterBuffer))
.set(BLOOM_FILTER_STATE_GENERATING_KEY, false)
.set(BLOOM_FILTER_STATE_EMPTY_BADGE_KEY, (emptyBadgeCount || 0).toString())
.set(BLOOM_FILTER_STATE_TOTAL_BADGE_KEY, (totalBadgeCount || 0).toString())
......@@ -146,6 +141,5 @@ worker.on('message', async bloomFilterArrayDump => {
module.exports = {
updateBloomFilter,
getBloomFilter,
getBloomFilterEtag,
getBloomFilterAndEtag,
};
......@@ -10,10 +10,10 @@ const {
HMAC_SHA256,
} = require('@lucaapp/crypto');
const redis = require('./redis');
const cache = require('./redisCache');
const database = require('../database');
const NOTIFICATIONS_CACHE_KEY = 'cache:notifications';
const NOTIFICATIONS_CACHE_KEY = 'notifications';
const {
DEVICE_TYPE_IOS,
DEVICE_TYPE_ANDROID,
......@@ -103,9 +103,10 @@ const generateNotifications = async () => {
}))
.filter(healthDepartment => healthDepartment.hashedTraceIds.length > 0);
redis.set(NOTIFICATIONS_CACHE_KEY, JSON.stringify(responseValue));
cache.set(NOTIFICATIONS_CACHE_KEY, JSON.stringify(responseValue));
};
const getNotifications = () => redis.get(NOTIFICATIONS_CACHE_KEY);
const getNotifications = () =>
cache.get(NOTIFICATIONS_CACHE_KEY).then(cachedResponse => cachedResponse[0]);
module.exports = { generateNotifications, getNotifications };
import etag from 'etag';
import { set as redisSet, get as redisGet } from './redis';
const getDataKey = (key: string) => `cache:data:${key}`;
const getEtagKey = (key: string) => `cache:etag:${key}`;
const cacheEtagStore = new Map();
const cacheDataStore = new Map();
const get = async (key: string, bufferStore = false) => {
const localEtag = cacheEtagStore.get(key);
const remoteEtag = await redisGet(getEtagKey(key));
if (localEtag === remoteEtag) {
return [cacheDataStore.get(key), localEtag];
}
// If the key is passed as buffer the response will also be a buffer.
const dataKey = bufferStore ? Buffer.from(getDataKey(key)) : getDataKey(key);
const remoteValue = await redisGet(dataKey);
cacheDataStore.set(key, remoteValue);
cacheEtagStore.set(key, remoteEtag);
return [remoteValue, remoteEtag];
};
const set = (key: string, value: string | Buffer) => {
const calculatedEtag = etag(value);
const dataKey = getDataKey(key);
redisSet(dataKey, value);
redisSet(getEtagKey(key), calculatedEtag);
cacheEtagStore.set(key, calculatedEtag);
cacheDataStore.set(key, value);
};
const cache = { get, set };
module.exports = cache;
......@@ -416,6 +416,13 @@
resolved "https://registry.yarnpkg.com/@types/cookiejar/-/cookiejar-2.1.2.tgz#66ad9331f63fe8a3d3d9d8c6e3906dd10f6446e8"
integrity sha512-t73xJJrvdTjXrn4jLS9VSGRbz0nUY3cl2DMGDU48lKl+HR9dbbjW2A9r3g40VA++mQpy6uuHg33gy7du2BKpog==
"@types/etag@1.8.1":
version "1.8.1"
resolved "https://registry.yarnpkg.com/@types/etag/-/etag-1.8.1.tgz#593ca8ddb43acb3db049bd0955fd64d281ab58b9"
integrity sha512-bsKkeSqN7HYyYntFRAmzcwx/dKW4Wa+KVMTInANlI72PWLQmOpZu96j0OqHZGArW4VQwCmJPteQlXaUDeOB0WQ==
dependencies:
"@types/node" "*"
"@types/expect@24.3.0":
version "24.3.0"
resolved "https://registry.yarnpkg.com/@types/expect/-/expect-24.3.0.tgz#d7cab8b3c10c2d92a0cbb31981feceb81d3486f1"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment