Unverified Commit bb159be3 authored by Philipp Berger's avatar Philipp Berger
Browse files

chore: release v1.5.4

parent 78eedf96
# Changelog
### 1.5.4 (2021-07-21)
* **scanner:** feat: deny checkins from unregistered badges via bloomfilter
### 1.5.3 (2021-07-16)
* **health-department:** feat: add utf-8 BOM for better excel compatibility
......
{
"name": "Using fixtures to represent data",
"email": "hello@cypress.io",
"body": "Fixtures are a great way to mock data for responses to routes"
}
\ No newline at end of file
{
"name": "e2e",
"version": "1.5.3",
"version": "1.5.4",
"main": "index.js",
"private": true,
"engines": {
......
import {
DELETED_REGISTERED_BADGE,
REGISTERED_BADGE,
UNREGISTERED_BADGE,
} from '../helpers/badges';
import { inputQRCodeData } from '../helpers/input';
import { regenerateBloomFilter } from '../helpers/api';
import {
visitScannerAndWaitForFilterLoad,
visitScanner,
} from '../helpers/intercept';
describe('Check bloom filter for unregistered and deleted badges', () => {
it('can check in a registered badge', () => {
regenerateBloomFilter();
visitScannerAndWaitForFilterLoad();
inputQRCodeData(REGISTERED_BADGE);
cy.getByCy('badgeCheckInSuccess').should('exist');
});
it('cannot check in a unregistered badge', () => {
visitScanner();
inputQRCodeData(UNREGISTERED_BADGE);
cy.get('.noBadgeUserDataError').should('exist');
});
it('cannot check in a deleted registered badge', () => {
visitScanner();
inputQRCodeData(DELETED_REGISTERED_BADGE);
cy.get('.noBadgeUserDataError').should('exist');
});
});
export const regenerateBloomFilter = () => {
cy.request(
'POST',
'https://localhost/api/internal/jobs/regenerateBloomFilter'
);
};
\ No newline at end of file
export const UNREGISTERED_BADGE =
'1oFQNNlaWM(:pX$B#$3Zs^[%iRlc%x6(ZOq=[4pTixOyTz&2ZysGNu7gbQ>m!ntV%!-L1DXSxo7k:pA4pNfH^jwDT!B)a7niO^e4tma{uSrZIUpEWMoG%XDN)m@PzJNam[Db%lFqI#@gz-#l<L{mYZR=sZbmj$-R]BvClf5s](@s5mJ^QZzGVUX+:{Om56';
export const DELETED_REGISTERED_BADGE =
'1oFQqEKYwu9eFBBSNkcPF:R<muW]lciJ6oP)Dy8eADpIu]C({u+VaT#)HDd?Lqr:LTh!2a>#/[>p$<Pu<*Ly-m)KsCGfNP]6Rjcib5Gat2c!*g(1J>r>}GKcWMyc]lZ>oZQIT>WCr6ZE:V#a0y1oDD^ZWyU]4CPq&OwKMo30jSM*+3uLReszSBg$9z@Gd&';
export const REGISTERED_BADGE =
'1oFPTDj7bJ!cCHw@{sdsV.Aq5le2.2Mph%XF@ZzsGk=:GV)rKC7jJA(k5cA5>#sO.o$HB!Is-/)WksDX3c!O!4FqYBoXcQM8wp/dhr)KwB=wNBD+5%>a9]):>KC-f^uX+2Kk<KN}88ufZnn:09:HvY)EueJF%2n?t=rG*2]i>a9x})-4uWhNNzpEHM&kG:';
\ No newline at end of file
export const inputQRCodeData = data => {
cy.get('input').type(data, {
parseSpecialCharSequences: false,
});
cy.get('input').type('{enter}');
};
\ No newline at end of file
import { SCANNER_ROUTE } from './routes';
export const interceptBloomFilterCall = () =>
cy.intercept({
method: 'GET',
url: 'https://localhost/api/v3/badges/bloomFilter',
});
export const visitScanner = () => cy.visit(SCANNER_ROUTE);
export const visitScannerAndWaitForFilterLoad = () => {
interceptBloomFilterCall().as('getBloomFilter');
visitScanner();
cy.wait('@getBloomFilter');
};
\ No newline at end of file
export const SCANNER_ROUTE = '/scanner/660582bd-73f3-4bbf-8570-969625218001';
\ No newline at end of file
{
"name": "@lucaapp/web",
"version": "1.5.3",
"version": "1.5.4",
"private": true,
"license": "Apache-2.0",
"author": "Culture4Life <hello@luca-app.de> (https://www.luca-app.de/)",
......
......@@ -105,6 +105,10 @@ module.exports = {
},
},
},
bloomFilter: {
// One in 100 million badge users
falsePositiveRate: 0.00000001,
},
certs: {
dtrust: {
root: `-----BEGIN CERTIFICATE-----
......
{
"name": "@lucaapp/backend",
"version": "1.5.3",
"version": "1.5.4",
"private": true,
"license": "Apache-2.0",
"author": "Culture4Life <hello@luca-app.de> (https://www.luca-app.de/)",
......@@ -24,12 +24,14 @@
"dependencies": {
"@lucaapp/crypto": "2.0.3",
"axios": "0.21.1",
"bloomit": "1.1.1",
"colors": "1.4.0",
"config": "3.3.6",
"connect-session-sequelize": "7.1.1",
"cookie-parser": "1.4.5",
"cors": "2.8.5",
"escape-html": "1.0.3",
"etag": "1.8.1",
"express": "4.17.1",
"express-async-errors": "3.1.1",
"express-rate-limit": "5.2.6",
......
......@@ -159,11 +159,35 @@ const groups = [
},
];
// This is junk data and is only used for the bloom filter test
const badges = [
{
uuid: 'c7258950-2fda-4ac5-b49c-68fb123a3bfa',
data: '',
deviceType: 'static',
publicKey: 'AiCjeREjiUOUqDcxtz/SfCIm7mvCZfKP1Tdr+TKUJ3a7',
},
{
uuid: '9184336d-930e-4b85-a2b3-4d303111fcba',
data: 'zgjnCzccRasREjLx1dnuex56mTN5QFvUGYGZOLWdzrib',
deviceType: 'static',
publicKey: 'A47A5rTlfYRcMdwFrAy8cJBBvH4V/ldVeUC/lofAca/u',
},
{
uuid: '2a4c47e3-834d-4c85-9335-2628cbbfca0f',
data: 'rM9pFtUU3dGyMjDCP1EonYHoVngdmafdIb8ThWPjHFJS',
deviceType: 'static',
publicKey: 'ArZP4Xyzg46XviignaTiOkufUPDRPd4Z08TuJ6BZNB8p',
deletedAt: '2021-06-09 12:27:26.056+00',
},
];
module.exports = {
up: async queryInterface => {
await queryInterface.bulkInsert('Operators', operators);
await queryInterface.bulkInsert('LocationGroups', groups);
await queryInterface.bulkInsert('Locations', locations);
await queryInterface.bulkInsert('Users', badges);
},
down: () => {
console.warn('Not implemented.');
......
......@@ -5,12 +5,14 @@ const router = require('express').Router();
const moment = require('moment');
const crypto = require('crypto');
const { Op, fn, col } = require('sequelize');
const status = require('http-status');
const { GET_RANDOM_BYTES, hexToBase64 } = require('@lucaapp/crypto');
const database = require('../../database');
const featureFlag = require('../../utils/featureFlag');
const { generateNotifications } = require('../../utils/notifications.js');
const { updateBloomFilter } = require('../../utils/bloomFilter.js');
router.post('/deleteOldTraces', async (request, response) => {
const t0 = performance.now();
......@@ -261,4 +263,9 @@ router.post('/regenerateNotifications', async (request, response) => {
});
});
router.post('/regenerateBloomFilter', async (request, response) => {
updateBloomFilter();
response.sendStatus(status.NO_CONTENT);
});
module.exports = router;
......@@ -14,6 +14,11 @@ const {
const { validateSchema } = require('../../middlewares/validateSchema');
const { limitRequestsPerHour } = require('../../middlewares/rateLimit');
const {
getBloomFilter,
getBloomFilterEtag,
} = require('../../utils/bloomFilter');
const database = require('../../database');
const { badgeCreateSchema } = require('./badges.schemas');
......@@ -70,4 +75,19 @@ router.post(
}
);
router.get('/bloomFilter', async (request, response) => {
const bloomFilterEtag = await getBloomFilterEtag();
if (bloomFilterEtag === request.headers['If-None-Match']) {
return response.sendStatus(status.NOT_MODIFIED);
}
const bloomFilter = await getBloomFilter();
if (!bloomFilter) {
return response.sendStatus(status.NOT_FOUND);
}
response.setHeader('ETag', bloomFilterEtag);
return response.send(bloomFilter);
});
module.exports = router;
paths:
/badges/bloomFilter:
get:
tags:
- Badges
responses:
'200':
description: OK
content:
application/octet-stream:
schema:
type: string
format: binary
'304':
description: Not Modified
'404':
description: Not Found
operationId: get-bloomFilter
description: Get a bloom filter that holds all unregistered badge UUIDs
parameters:
- schema:
type: string
name: If-None-Match
in: header
required: true
/badges/:
post:
summary: Create badge user
......
const moment = require('moment');
const etag = require('etag');
const { Worker } = require('worker_threads');
const { Op } = require('sequelize');
const logger = require('./logger');
const lifecycle = require('./lifecycle');
const {
client: redisClient,
set: redisSet,
get: redisGet,
} = require('./redis');
const database = require('../database');
const BLOOM_FILTER_ETAG_KEY = 'BadgeBloomFilterEtag';
const BLOOM_FILTER_BUFFER_KEY = 'BadgeBloomFilterBuffer';
const BLOOM_FILTER_STATE_GENERATING_KEY = 'IsBloomFilterGenerating';
const BLOOM_FILTER_STATE_EMPTY_BADGE_KEY = 'LastEmptyBadgeCount';
const BLOOM_FILTER_STATE_TOTAL_BADGE_KEY = 'LastTotalBadgeCount';
const EXPIRATION_MULTIPLIER_MS = 2;
const worker = new Worker(`${__dirname}/bloomFilter.worker.js`);
worker.unref();
lifecycle.registerShutdownHandler(() => worker.terminate());
const getIsBloomFilterGenerating = () =>
redisGet(BLOOM_FILTER_STATE_GENERATING_KEY)
.then(value => value === 'true')
.catch(() => false);
const getLastEmptyBadgeCount = () =>
redisGet(BLOOM_FILTER_STATE_EMPTY_BADGE_KEY)
.then(value => Number.parseInt(value, 10))
.catch(() => 0);
const getLastTotalBadgeCount = () =>
redisGet(BLOOM_FILTER_STATE_TOTAL_BADGE_KEY)
.then(value => Number.parseInt(value, 10))
.catch(() => 0);
const getEmptyBadgeCount = async () =>
database.User.count({
where: {
deviceType: 'static',
[Op.or]: [
{
data: {
[Op.eq]: '',
},
},
{
deletedAt: {
[Op.not]: null,
},
},
],
},
paranoid: false,
});
const getUnregisteredBadges = async () =>
database.User.findAll({
where: {
[Op.or]: [
{
data: {
[Op.eq]: '',
},
},
{
deletedAt: {
[Op.not]: null,
},
},
],
deviceType: 'static',
},
attributes: ['uuid'],
raw: true,
paranoid: false,
});
const getTotalBadgeCount = async () =>
database.User.count({
where: {
deviceType: 'static',
},
paranoid: false,
});
const needsToUpdate = async () => {
const emptyBadgeCount = (await getEmptyBadgeCount()) || 0;
const totalBadgeCount = (await getTotalBadgeCount()) || 0;
if (
(await getLastEmptyBadgeCount()) === emptyBadgeCount &&
(await getLastTotalBadgeCount()) === totalBadgeCount
) {
return false;
}
return true;
};
const updateBloomFilter = async () => {
if (await getIsBloomFilterGenerating()) {
logger.info('Bloom filter is currently generating');
return;
}
if (!(await needsToUpdate())) {
logger.info('Bloom filter is up to date');
return;
}
const emptyBadgeCount = await getEmptyBadgeCount();
logger.info(`Bloom filter is now generating for ${emptyBadgeCount} badges.`);
const expirationTime = Math.max(
moment.duration(1, 'minute').asMilliseconds(),
EXPIRATION_MULTIPLIER_MS * emptyBadgeCount
);
await redisSet(BLOOM_FILTER_STATE_GENERATING_KEY, true, 'PX', expirationTime);
worker.postMessage(await getUnregisteredBadges());
};
const getBloomFilter = () =>
redisGet(Buffer.from(BLOOM_FILTER_BUFFER_KEY)).catch(() => null);
const getBloomFilterEtag = () => redisGet(BLOOM_FILTER_ETAG_KEY);
worker.on('message', async bloomFilterArrayDump => {
const emptyBadgeCount = await getEmptyBadgeCount();
const totalBadgeCount = await getTotalBadgeCount();
const bloomFilterBuffer = Buffer.from(bloomFilterArrayDump);
redisClient
.multi()
.set(BLOOM_FILTER_BUFFER_KEY, bloomFilterBuffer)
.set(BLOOM_FILTER_ETAG_KEY, etag(bloomFilterBuffer))
.set(BLOOM_FILTER_STATE_GENERATING_KEY, false)
.set(BLOOM_FILTER_STATE_EMPTY_BADGE_KEY, (emptyBadgeCount || 0).toString())
.set(BLOOM_FILTER_STATE_TOTAL_BADGE_KEY, (totalBadgeCount || 0).toString())
.exec();
logger.info('Bloom filter generated');
});
module.exports = {
updateBloomFilter,
getBloomFilter,
getBloomFilterEtag,
};
/* eslint no-underscore-dangle: 0 */
const config = require('config');
const { parentPort } = require('worker_threads');
const { BloomFilter } = require('bloomit');
const { SHA256, uuidToHex } = require('@lucaapp/crypto');
const FALSE_POSITIVE_RATE = config.get('bloomFilter.falsePositiveRate');
const unregisteredBadgeBloomFilter = async unregisteredBadges => {
const filter = BloomFilter.create(
unregisteredBadges.length,
FALSE_POSITIVE_RATE
);
unregisteredBadges.forEach(badge => {
const uuidSHA256 = SHA256(uuidToHex(badge.uuid));
filter.add(uuidSHA256);
});
return filter.export();
};
parentPort.on('message', async unregisteredBadges => {
const bloomFilter = await unregisteredBadgeBloomFilter(unregisteredBadges);
parentPort.postMessage(bloomFilter);
});
......@@ -668,6 +668,13 @@ block-stream@*:
dependencies:
inherits "~2.0.0"
bloomit@1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/bloomit/-/bloomit-1.1.1.tgz#23e9cadc8a50f6cda9074c64c22bc5589aa50853"
integrity sha512-gd6otAsteYqZ9SE3r37HmXiAN9MiRXzP86fNv96GCV/GLbNa/aBizhWOCNr/6quy9+Kyu8AEUE4yJyAOK4kJrw==
dependencies:
xxhashjs "^0.2.2"
bluebird@^3.7.2:
version "3.7.2"
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
......@@ -1209,6 +1216,11 @@ crypto-random-string@^2.0.0:
resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-2.0.0.tgz#ef2a7a966ec11083388369baa02ebead229b30d5"
integrity sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==
cuint@^0.2.2:
version "0.2.2"
resolved "https://registry.yarnpkg.com/cuint/-/cuint-0.2.2.tgz#408086d409550c2631155619e9fa7bcadc3b991b"
integrity sha1-QICG1AlVDCYxFVYZ6fp7ytw7mRs=
d@1, d@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/d/-/d-1.0.1.tgz#8698095372d58dbee346ffd0c7093f99f8f9eb5a"
......@@ -1852,7 +1864,7 @@ esutils@^2.0.2:
resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64"
integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==
etag@~1.8.1:
etag@1.8.1, etag@~1.8.1:
version "1.8.1"
resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887"
integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=
......@@ -5530,6 +5542,13 @@ xtend@^4.0.0:
resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"
integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==
xxhashjs@^0.2.2:
version "0.2.2"
resolved "https://registry.yarnpkg.com/xxhashjs/-/xxhashjs-0.2.2.tgz#8a6251567621a1c46a5ae204da0249c7f8caa9d8"
integrity sha512-AkTuIuVTET12tpsVIQo+ZU6f/qDmKuRUcjaqR+OIvm+aCBsZ95i7UVY5WJ9TMsSaZ0DA2WxoZ4acu0sPH+OKAw==
dependencies:
cuint "^0.2.2"
y18n@^4.0.0:
version "4.0.3"
resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.3.tgz#b5f259c82cd6e336921efd7bfd8bf560de9eeedf"
......
{
"name": "@lucaapp/contact-form",
"version": "1.5.3",
"version": "1.5.4",
"private": true,
"license": "Apache-2.0",
"author": "Culture4Life <hello@luca-app.de> (https://www.luca-app.de/)",
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment