Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
Felix Foertsch
Luca Web
Commits
c1d9883b
Unverified
Commit
c1d9883b
authored
Jul 28, 2021
by
Philipp Berger
Browse files
chore: release v1.6.1
parent
21fdea04
Changes
7
Hide whitespace changes
Inline
Side-by-side
CHANGELOG.md
View file @
c1d9883b
# Changelog
### 1.6.1 (2021-07-28)
*
**backend:**
feat: lower redis usage by adding etag caching for large values
### 1.6.0 (2021-07-27)
*
**backend:**
fix: schema validation mismatch
*
**backend:**
feat: added dev setup for signing tool
...
...
services/backend/package.json
View file @
c1d9883b
...
...
@@ -26,6 +26,7 @@
"@lucaapp/crypto"
:
"2.0.3"
,
"@tsconfig/node14"
:
"1.0.1"
,
"@types/config"
:
"0.0.39"
,
"@types/etag"
:
"1.8.1"
,
"@types/express"
:
"4.17.13"
,
"@types/validator"
:
"13.6.3"
,
"axios"
:
"0.21.1"
,
...
...
services/backend/src/routes/v3/badges.js
View file @
c1d9883b
...
...
@@ -14,10 +14,7 @@ const {
const
{
validateSchema
}
=
require
(
'
../../middlewares/validateSchema
'
);
const
{
limitRequestsPerHour
}
=
require
(
'
../../middlewares/rateLimit
'
);
const
{
getBloomFilter
,
getBloomFilterEtag
,
}
=
require
(
'
../../utils/bloomFilter
'
);
const
{
getBloomFilterAndEtag
}
=
require
(
'
../../utils/bloomFilter
'
);
const
database
=
require
(
'
../../database
'
);
const
{
badgeCreateSchema
}
=
require
(
'
./badges.schemas
'
);
...
...
@@ -78,12 +75,11 @@ router.post(
);
router
.
get
(
'
/bloomFilter
'
,
async
(
request
,
response
)
=>
{
const
bloomFilterEtag
=
await
getBloomFilterEtag
();
const
[
bloomFilter
,
bloomFilterEtag
]
=
await
getBloomFilter
And
Etag
();
if
(
bloomFilterEtag
===
request
.
headers
[
'
If-None-Match
'
])
{
return
response
.
sendStatus
(
status
.
NOT_MODIFIED
);
}
const
bloomFilter
=
await
getBloomFilter
();
if
(
!
bloomFilter
)
{
return
response
.
sendStatus
(
status
.
NOT_FOUND
);
}
...
...
services/backend/src/utils/bloomFilter.js
View file @
c1d9883b
const
moment
=
require
(
'
moment
'
);
const
etag
=
require
(
'
etag
'
);
const
{
Worker
}
=
require
(
'
worker_threads
'
);
const
{
Op
}
=
require
(
'
sequelize
'
);
...
...
@@ -10,9 +9,9 @@ const {
set
:
redisSet
,
get
:
redisGet
,
}
=
require
(
'
./redis
'
);
const
cache
=
require
(
'
./redisCache
'
);
const
database
=
require
(
'
../database
'
);
const
BLOOM_FILTER_ETAG_KEY
=
'
BadgeBloomFilterEtag
'
;
const
BLOOM_FILTER_BUFFER_KEY
=
'
BadgeBloomFilterBuffer
'
;
const
BLOOM_FILTER_STATE_GENERATING_KEY
=
'
IsBloomFilterGenerating
'
;
const
BLOOM_FILTER_STATE_EMPTY_BADGE_KEY
=
'
LastEmptyBadgeCount
'
;
...
...
@@ -123,19 +122,15 @@ const updateBloomFilter = async () => {
worker
.
postMessage
(
await
getUnregisteredBadges
());
};
const
getBloomFilter
=
()
=>
redisGet
(
Buffer
.
from
(
BLOOM_FILTER_BUFFER_KEY
)).
catch
(()
=>
null
);
const
getBloomFilterEtag
=
()
=>
redisGet
(
BLOOM_FILTER_ETAG_KEY
);
const
getBloomFilterAndEtag
=
()
=>
cache
.
get
(
BLOOM_FILTER_BUFFER_KEY
,
true
);
worker
.
on
(
'
message
'
,
async
bloomFilterArrayDump
=>
{
const
emptyBadgeCount
=
await
getEmptyBadgeCount
();
const
totalBadgeCount
=
await
getTotalBadgeCount
();
const
bloomFilterBuffer
=
Buffer
.
from
(
bloomFilterArrayDump
);
cache
.
set
(
BLOOM_FILTER_BUFFER_KEY
,
bloomFilterBuffer
);
redisClient
.
multi
()
.
set
(
BLOOM_FILTER_BUFFER_KEY
,
bloomFilterBuffer
)
.
set
(
BLOOM_FILTER_ETAG_KEY
,
etag
(
bloomFilterBuffer
))
.
set
(
BLOOM_FILTER_STATE_GENERATING_KEY
,
false
)
.
set
(
BLOOM_FILTER_STATE_EMPTY_BADGE_KEY
,
(
emptyBadgeCount
||
0
).
toString
())
.
set
(
BLOOM_FILTER_STATE_TOTAL_BADGE_KEY
,
(
totalBadgeCount
||
0
).
toString
())
...
...
@@ -146,6 +141,5 @@ worker.on('message', async bloomFilterArrayDump => {
module
.
exports
=
{
updateBloomFilter
,
getBloomFilter
,
getBloomFilterEtag
,
getBloomFilterAndEtag
,
};
services/backend/src/utils/notifications.js
View file @
c1d9883b
...
...
@@ -10,10 +10,10 @@ const {
HMAC_SHA256
,
}
=
require
(
'
@lucaapp/crypto
'
);
const
redis
=
require
(
'
./redis
'
);
const
cache
=
require
(
'
./redis
Cache
'
);
const
database
=
require
(
'
../database
'
);
const
NOTIFICATIONS_CACHE_KEY
=
'
cache:
notifications
'
;
const
NOTIFICATIONS_CACHE_KEY
=
'
notifications
'
;
const
{
DEVICE_TYPE_IOS
,
DEVICE_TYPE_ANDROID
,
...
...
@@ -103,9 +103,10 @@ const generateNotifications = async () => {
}))
.
filter
(
healthDepartment
=>
healthDepartment
.
hashedTraceIds
.
length
>
0
);
redis
.
set
(
NOTIFICATIONS_CACHE_KEY
,
JSON
.
stringify
(
responseValue
));
cache
.
set
(
NOTIFICATIONS_CACHE_KEY
,
JSON
.
stringify
(
responseValue
));
};
const
getNotifications
=
()
=>
redis
.
get
(
NOTIFICATIONS_CACHE_KEY
);
const
getNotifications
=
()
=>
cache
.
get
(
NOTIFICATIONS_CACHE_KEY
).
then
(
cachedResponse
=>
cachedResponse
[
0
]);
module
.
exports
=
{
generateNotifications
,
getNotifications
};
services/backend/src/utils/redisCache.ts
0 → 100644
View file @
c1d9883b
import
etag
from
'
etag
'
;
import
{
set
as
redisSet
,
get
as
redisGet
}
from
'
./redis
'
;
const
getDataKey
=
(
key
:
string
)
=>
`cache:data:
${
key
}
`
;
const
getEtagKey
=
(
key
:
string
)
=>
`cache:etag:
${
key
}
`
;
const
cacheEtagStore
=
new
Map
();
const
cacheDataStore
=
new
Map
();
const
get
=
async
(
key
:
string
,
bufferStore
=
false
)
=>
{
const
localEtag
=
cacheEtagStore
.
get
(
key
);
const
remoteEtag
=
await
redisGet
(
getEtagKey
(
key
));
if
(
localEtag
===
remoteEtag
)
{
return
[
cacheDataStore
.
get
(
key
),
localEtag
];
}
// If the key is passed as buffer the response will also be a buffer.
const
dataKey
=
bufferStore
?
Buffer
.
from
(
getDataKey
(
key
))
:
getDataKey
(
key
);
const
remoteValue
=
await
redisGet
(
dataKey
);
cacheDataStore
.
set
(
key
,
remoteValue
);
cacheEtagStore
.
set
(
key
,
remoteEtag
);
return
[
remoteValue
,
remoteEtag
];
};
const
set
=
(
key
:
string
,
value
:
string
|
Buffer
)
=>
{
const
calculatedEtag
=
etag
(
value
);
const
dataKey
=
getDataKey
(
key
);
redisSet
(
dataKey
,
value
);
redisSet
(
getEtagKey
(
key
),
calculatedEtag
);
cacheEtagStore
.
set
(
key
,
calculatedEtag
);
cacheDataStore
.
set
(
key
,
value
);
};
const
cache
=
{
get
,
set
};
module
.
exports
=
cache
;
services/backend/yarn.lock
View file @
c1d9883b
...
...
@@ -416,6 +416,13 @@
resolved "https://registry.yarnpkg.com/@types/cookiejar/-/cookiejar-2.1.2.tgz#66ad9331f63fe8a3d3d9d8c6e3906dd10f6446e8"
integrity sha512-t73xJJrvdTjXrn4jLS9VSGRbz0nUY3cl2DMGDU48lKl+HR9dbbjW2A9r3g40VA++mQpy6uuHg33gy7du2BKpog==
"@types/etag@1.8.1":
version "1.8.1"
resolved "https://registry.yarnpkg.com/@types/etag/-/etag-1.8.1.tgz#593ca8ddb43acb3db049bd0955fd64d281ab58b9"
integrity sha512-bsKkeSqN7HYyYntFRAmzcwx/dKW4Wa+KVMTInANlI72PWLQmOpZu96j0OqHZGArW4VQwCmJPteQlXaUDeOB0WQ==
dependencies:
"@types/node" "*"
"@types/expect@24.3.0":
version "24.3.0"
resolved "https://registry.yarnpkg.com/@types/expect/-/expect-24.3.0.tgz#d7cab8b3c10c2d92a0cbb31981feceb81d3486f1"
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment