From 9ebe5c18c5e26ad720e9661bc6677e01ca0b3c65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakob=20M=C3=B6ller?= <jakob.moeller@sap.com>
Date: Mon, 7 Sep 2020 16:03:28 +0200
Subject: [PATCH] Remove Cache Clearing to reduce CDN load on failing devices
(EXPOSUREAPP-2405) (#1108)
* Remove Cache Clearing to reduce CDN load on failing devices. This ideally needs to be accompanied by a way to clear the cache manually and a way to identify more root causes of Transaction Failures.
Signed-off-by: d067928 <jakob.moeller@sap.com>
* Remove Files that failed for Key Retrieval
Only Remove Files from Cache that failed to download, instead of every file. This is accompanied ideally by no rollback in the Key Retrieval. We only delete the cache ref, the file will not be deleted as it is considered not present
Signed-off-by: d067928 <jakob.moeller@sap.com>
* Introduce dedicated QuotaCalculator for Unit Testing
Signed-off-by: d067928 <jakob.moeller@sap.com>
* Refactor QuotaCalculator for LocalData Property Access and Write Tests
Signed-off-by: d067928 <jakob.moeller@sap.com>
* Use Instant on the Device Read since this is not timezone specific
Signed-off-by: d067928 <jakob.moeller@sap.com>
* Add specific state for the quota calculation and dedicated rollback
Signed-off-by: d067928 <jakob.moeller@sap.com>
* PR Comments
Signed-off-by: d067928 <jakob.moeller@sap.com>
---
.../coronawarnapp/http/WebRequestBuilder.kt | 7 +-
.../de/rki/coronawarnapp/storage/LocalData.kt | 40 +++
.../storage/keycache/KeyCacheDao.kt | 6 +
.../storage/keycache/KeyCacheRepository.kt | 8 +
.../RetrieveDiagnosisKeysTransaction.kt | 57 +++-
.../coronawarnapp/util/CachedKeyFileHolder.kt | 29 +-
.../util/GoogleQuotaCalculator.kt | 62 ++++
.../rki/coronawarnapp/util/QuotaCalculator.kt | 29 ++
.../RetrieveDiagnosisKeysTransactionTest.kt | 7 +
.../util/CachedKeyFileHolderTest.kt | 3 +
.../util/GoogleQuotaCalculatorTest.kt | 295 ++++++++++++++++++
11 files changed, 523 insertions(+), 20 deletions(-)
create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/GoogleQuotaCalculator.kt
create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/QuotaCalculator.kt
create mode 100644 Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/GoogleQuotaCalculatorTest.kt
diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/WebRequestBuilder.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/WebRequestBuilder.kt
index b6dc003eb..906b6b44c 100644
--- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/WebRequestBuilder.kt
+++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/WebRequestBuilder.kt
@@ -102,9 +102,12 @@ class WebRequestBuilder(
): File = withContext(Dispatchers.IO) {
val fileName = "${UUID.nameUUIDFromBytes(url.toByteArray())}.zip"
val file = File(FileStorageHelper.keyExportDirectory, fileName)
- file.outputStream().use {
+ if (file.exists()) file.delete()
+ file.outputStream().use { fos ->
Timber.v("Added $url to queue.")
- distributionService.getKeyFiles(url).byteStream().copyTo(it, DEFAULT_BUFFER_SIZE)
+ distributionService.getKeyFiles(url).byteStream().use {
+ it.copyTo(fos, DEFAULT_BUFFER_SIZE)
+ }
Timber.v("key file request successful.")
}
return@withContext file
diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/LocalData.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/LocalData.kt
index 889ab5669..48bc37e40 100644
--- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/LocalData.kt
+++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/LocalData.kt
@@ -6,6 +6,7 @@ import de.rki.coronawarnapp.CoronaWarnApplication
import de.rki.coronawarnapp.R
import de.rki.coronawarnapp.risk.RiskLevel
import de.rki.coronawarnapp.util.security.SecurityHelper.globalEncryptedSharedPreferencesInstance
+import org.joda.time.Instant
import java.util.Date
/**
@@ -18,6 +19,11 @@ object LocalData {
private val TAG: String? = LocalData::class.simpleName
+ private const val PREFERENCE_NEXT_TIME_RATE_LIMITING_UNLOCKS =
+ "preference_next_time_rate_limiting_unlocks"
+ private const val PREFERENCE_GOOGLE_API_PROVIDE_DIAGNOSIS_KEYS_CALL_COUNT =
+ "preference_google_api_provide_diagnosis_keys_call_count"
+
/****************************************************
* ONBOARDING DATA
****************************************************/
@@ -390,6 +396,40 @@ object LocalData {
}
}
+ var nextTimeRateLimitingUnlocks: Instant
+ get() {
+ return Instant.ofEpochMilli(
+ getSharedPreferenceInstance().getLong(
+ PREFERENCE_NEXT_TIME_RATE_LIMITING_UNLOCKS,
+ 0L
+ )
+ )
+ }
+ set(value) {
+ getSharedPreferenceInstance().edit(true) {
+ putLong(
+ PREFERENCE_NEXT_TIME_RATE_LIMITING_UNLOCKS,
+ value.millis
+ )
+ }
+ }
+
+ var googleAPIProvideDiagnosisKeysCallCount: Int
+ get() {
+ return getSharedPreferenceInstance().getInt(
+ PREFERENCE_GOOGLE_API_PROVIDE_DIAGNOSIS_KEYS_CALL_COUNT,
+ 0
+ )
+ }
+ set(value) {
+ getSharedPreferenceInstance().edit(true) {
+ putInt(
+ PREFERENCE_GOOGLE_API_PROVIDE_DIAGNOSIS_KEYS_CALL_COUNT,
+ value
+ )
+ }
+ }
+
/**
* Gets the last time of successful risk level calculation as long
* from the EncryptedSharedPrefs
diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/keycache/KeyCacheDao.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/keycache/KeyCacheDao.kt
index ec812964a..0e6cbeadb 100644
--- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/keycache/KeyCacheDao.kt
+++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/keycache/KeyCacheDao.kt
@@ -35,6 +35,9 @@ interface KeyCacheDao {
@Query("SELECT * FROM date")
suspend fun getAllEntries(): List<KeyCacheEntity>
+ @Query("SELECT * FROM date WHERE id IN (:idList)")
+ suspend fun getAllEntries(idList: List<String>): List<KeyCacheEntity>
+
@Query("DELETE FROM date")
suspend fun clear()
@@ -44,6 +47,9 @@ interface KeyCacheDao {
@Delete
suspend fun deleteEntry(entity: KeyCacheEntity)
+ @Delete
+ suspend fun deleteEntries(entities: List<KeyCacheEntity>)
+
@Insert
suspend fun insertEntry(keyCacheEntity: KeyCacheEntity): Long
}
diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/keycache/KeyCacheRepository.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/keycache/KeyCacheRepository.kt
index 9e3ce0003..dd6b6425d 100644
--- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/keycache/KeyCacheRepository.kt
+++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/keycache/KeyCacheRepository.kt
@@ -88,6 +88,14 @@ class KeyCacheRepository(private val keyCacheDao: KeyCacheDao) {
keyCacheDao.clear()
}
+ suspend fun clear(idList: List<String>) {
+ if (idList.isNotEmpty()) {
+ val entries = keyCacheDao.getAllEntries(idList)
+ entries.forEach { deleteFileForEntry(it) }
+ keyCacheDao.deleteEntries(entries)
+ }
+ }
+
suspend fun getFilesFromEntries() = keyCacheDao
.getAllEntries()
.map { File(it.path) }
diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt
index 37f736283..20f3a38ee 100644
--- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt
+++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt
@@ -20,26 +20,29 @@
package de.rki.coronawarnapp.transaction
import com.google.android.gms.nearby.exposurenotification.ExposureConfiguration
-import de.rki.coronawarnapp.CoronaWarnApplication
import de.rki.coronawarnapp.nearby.InternalExposureNotificationClient
import de.rki.coronawarnapp.service.applicationconfiguration.ApplicationConfigurationService
import de.rki.coronawarnapp.storage.FileStorageHelper
import de.rki.coronawarnapp.storage.LocalData
-import de.rki.coronawarnapp.storage.keycache.KeyCacheRepository
import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.RetrieveDiagnosisKeysTransactionState.API_SUBMISSION
import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.RetrieveDiagnosisKeysTransactionState.CLOSE
import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.RetrieveDiagnosisKeysTransactionState.FETCH_DATE_UPDATE
import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.RetrieveDiagnosisKeysTransactionState.FILES_FROM_WEB_REQUESTS
+import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.RetrieveDiagnosisKeysTransactionState.QUOTA_CALCULATION
import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.RetrieveDiagnosisKeysTransactionState.RETRIEVE_RISK_SCORE_PARAMS
import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.RetrieveDiagnosisKeysTransactionState.SETUP
import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.RetrieveDiagnosisKeysTransactionState.TOKEN
import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.rollback
import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.start
import de.rki.coronawarnapp.util.CachedKeyFileHolder
+import de.rki.coronawarnapp.util.GoogleQuotaCalculator
+import de.rki.coronawarnapp.util.QuotaCalculator
import de.rki.coronawarnapp.worker.BackgroundWorkHelper
import org.joda.time.DateTime
import org.joda.time.DateTimeZone
+import org.joda.time.Duration
import org.joda.time.Instant
+import org.joda.time.chrono.GJChronology
import timber.log.Timber
import java.io.File
import java.util.Date
@@ -90,6 +93,9 @@ object RetrieveDiagnosisKeysTransaction : Transaction() {
/** Initial Setup of the Transaction and Transaction ID Generation and Date Lock */
SETUP,
+ /** calculates the Quota so that the rate limiting is caught gracefully*/
+ QUOTA_CALCULATION,
+
/** Initialisation of the identifying token used during the entire transaction */
TOKEN,
@@ -118,12 +124,25 @@ object RetrieveDiagnosisKeysTransaction : Transaction() {
/** atomic reference for the rollback value for created files during the transaction */
private val exportFilesForRollback = AtomicReference<List<File>>()
+ private val progressTowardsQuotaForRollback = AtomicReference<Int>()
+
+ private const val QUOTA_RESET_PERIOD_IN_HOURS = 24
+
+ private val quotaCalculator: QuotaCalculator<Int> = GoogleQuotaCalculator(
+ incrementByAmount = 14,
+ quotaLimit = 20,
+ quotaResetPeriod = Duration.standardHours(QUOTA_RESET_PERIOD_IN_HOURS.toLong()),
+ quotaTimeZone = DateTimeZone.UTC,
+ quotaChronology = GJChronology.getInstanceUTC()
+ )
+
suspend fun startWithConstraints() {
val currentDate = DateTime(Instant.now(), DateTimeZone.UTC)
val lastFetch = DateTime(
LocalData.lastTimeDiagnosisKeysFromServerFetch(),
DateTimeZone.UTC
)
+
if (LocalData.lastTimeDiagnosisKeysFromServerFetch() == null ||
currentDate.withTimeAtStartOfDay() != lastFetch.withTimeAtStartOfDay()
) {
@@ -152,6 +171,18 @@ object RetrieveDiagnosisKeysTransaction : Transaction() {
****************************************************/
val currentDate = executeSetup()
+ /****************************************************
+ * CALCULATE QUOTA FOR PROVIDE DIAGNOSIS KEYS
+ ****************************************************/
+ val hasExceededQuota = executeQuotaCalculation()
+
+ // When we are above the Quote, cancel the execution entirely
+ if (hasExceededQuota) {
+ Timber.tag(TAG).w("above quota, skipping RetrieveDiagnosisKeys")
+ executeClose()
+ return@lockAndExecuteUnique
+ }
+
/****************************************************
* RETRIEVE TOKEN
****************************************************/
@@ -194,8 +225,9 @@ object RetrieveDiagnosisKeysTransaction : Transaction() {
if (TOKEN.isInStateStack()) {
rollbackToken()
}
- if (FILES_FROM_WEB_REQUESTS.isInStateStack()) {
- rollbackFilesFromWebRequests()
+ // we reset the quota only if the submission has not happened yet
+ if (QUOTA_CALCULATION.isInStateStack() && !API_SUBMISSION.isInStateStack()) {
+ rollbackProgressTowardsQuota()
}
} catch (e: Exception) {
// We handle every exception through a RollbackException to make sure that a single EntryPoint
@@ -214,10 +246,9 @@ object RetrieveDiagnosisKeysTransaction : Transaction() {
LocalData.googleApiToken(googleAPITokenForRollback.get())
}
- private suspend fun rollbackFilesFromWebRequests() {
- Timber.v("rollback $FILES_FROM_WEB_REQUESTS")
- KeyCacheRepository.getDateRepository(CoronaWarnApplication.getAppContext())
- .clear()
+ private fun rollbackProgressTowardsQuota() {
+ Timber.v("rollback $QUOTA_CALCULATION")
+ quotaCalculator.resetProgressTowardsQuota(progressTowardsQuotaForRollback.get())
}
/**
@@ -230,6 +261,16 @@ object RetrieveDiagnosisKeysTransaction : Transaction() {
currentDate
}
+ /**
+ * Executes the QUOTA_CALCULATION Transaction State
+ */
+ private suspend fun executeQuotaCalculation() = executeState(
+ QUOTA_CALCULATION
+ ) {
+ progressTowardsQuotaForRollback.set(quotaCalculator.getProgressTowardsQuota())
+ quotaCalculator.calculateQuota()
+ }
+
/**
* Executes the TOKEN Transaction State
*/
diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/CachedKeyFileHolder.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/CachedKeyFileHolder.kt
index 15437cf47..97998f348 100644
--- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/CachedKeyFileHolder.kt
+++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/CachedKeyFileHolder.kt
@@ -37,6 +37,7 @@ import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.withContext
import timber.log.Timber
import java.io.File
+import java.util.Collections
import java.util.Date
import java.util.UUID
@@ -97,22 +98,30 @@ object CachedKeyFileHolder {
val deferredQueries: MutableCollection<Deferred<Any>> = mutableListOf()
keyCache.deleteOutdatedEntries(uuidListFromServer)
val missingDays = getMissingDaysFromDiff(serverDates)
+ val failedEntryCacheKeys = Collections.synchronizedList(mutableListOf<String>())
if (missingDays.isNotEmpty()) {
// we have a date difference
deferredQueries.addAll(
missingDays
.map { getURLForDay(it) }
- .map { url -> async { url.createDayEntryForUrl() } }
+ .map { url ->
+ val cacheKey = url.generateCacheKeyFromString()
+ async {
+ try {
+ url.createDayEntryForUrl(cacheKey)
+ } catch (e: Exception) {
+ Timber.v("failed entry: $cacheKey")
+ failedEntryCacheKeys.add(cacheKey)
+ }
+ }
+ }
)
}
// execute the query plan
- try {
- deferredQueries.awaitAll()
- } catch (e: Exception) {
- // For an error we clear the cache to try again
- keyCache.clear()
- throw e
- }
+ deferredQueries.awaitAll()
+ Timber.v("${failedEntryCacheKeys.size} failed entries ")
+ // For an error we clear the cache to try again
+ keyCache.clear(failedEntryCacheKeys)
keyCache.getFilesFromEntries()
.also { it.forEach { file -> Timber.v("cached file:${file.path}") } }
}
@@ -161,8 +170,8 @@ object CachedKeyFileHolder {
* Creates a date entry in the Key Cache for a given String with a unique Key Name derived from the URL
* and the URI of the downloaded File for that given key
*/
- private suspend fun String.createDayEntryForUrl() = keyCache.createEntry(
- this.generateCacheKeyFromString(),
+ private suspend fun String.createDayEntryForUrl(cacheKey: String) = keyCache.createEntry(
+ cacheKey,
WebRequestBuilder.getInstance().asyncGetKeyFilesFromServer(this).toURI(),
DAY
)
diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/GoogleQuotaCalculator.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/GoogleQuotaCalculator.kt
new file mode 100644
index 000000000..e7aace190
--- /dev/null
+++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/GoogleQuotaCalculator.kt
@@ -0,0 +1,62 @@
+package de.rki.coronawarnapp.util
+
+import de.rki.coronawarnapp.storage.LocalData
+import org.joda.time.Chronology
+import org.joda.time.DateTime
+import org.joda.time.DateTimeZone
+import org.joda.time.Duration
+import org.joda.time.Instant
+
+/**
+ * This Calculator class takes multiple parameters to check if the Google API
+ * can be called or the Rate Limit has been reached. The Quota is expected to reset at
+ * the start of the day in the given timeZone and Chronology
+ *
+ * @property incrementByAmount The amount of Quota Calls to increment per Call
+ * @property quotaLimit The maximum amount of Quota Calls allowed before Rate Limiting
+ * @property quotaResetPeriod The Period after which the Quota Resets
+ * @property quotaTimeZone The Timezone to work in
+ * @property quotaChronology The Chronology to work in
+ */
+class GoogleQuotaCalculator(
+ val incrementByAmount: Int,
+ val quotaLimit: Int,
+ val quotaResetPeriod: Duration,
+ val quotaTimeZone: DateTimeZone,
+ val quotaChronology: Chronology
+) : QuotaCalculator<Int> {
+ override var hasExceededQuota: Boolean = false
+
+ override fun calculateQuota(): Boolean {
+ if (Instant.now().isAfter(LocalData.nextTimeRateLimitingUnlocks)) {
+ LocalData.nextTimeRateLimitingUnlocks = DateTime
+ .now(quotaTimeZone)
+ .withChronology(quotaChronology)
+ .plus(quotaResetPeriod)
+ .withTimeAtStartOfDay()
+ .toInstant()
+ LocalData.googleAPIProvideDiagnosisKeysCallCount = 0
+ }
+
+ if (LocalData.googleAPIProvideDiagnosisKeysCallCount <= quotaLimit) {
+ LocalData.googleAPIProvideDiagnosisKeysCallCount += incrementByAmount
+ }
+
+ hasExceededQuota = LocalData.googleAPIProvideDiagnosisKeysCallCount > quotaLimit
+
+ return hasExceededQuota
+ }
+
+ override fun resetProgressTowardsQuota(newProgress: Int) {
+ if (newProgress > quotaLimit) {
+ throw IllegalArgumentException("cannot reset progress to a value higher than the quota limit")
+ }
+ if (newProgress % incrementByAmount != 0) {
+ throw IllegalArgumentException("supplied progress is no multiple of $incrementByAmount")
+ }
+ LocalData.googleAPIProvideDiagnosisKeysCallCount = newProgress
+ hasExceededQuota = false
+ }
+
+ override fun getProgressTowardsQuota(): Int = LocalData.googleAPIProvideDiagnosisKeysCallCount
+}
diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/QuotaCalculator.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/QuotaCalculator.kt
new file mode 100644
index 000000000..682f4a600
--- /dev/null
+++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/QuotaCalculator.kt
@@ -0,0 +1,29 @@
+package de.rki.coronawarnapp.util
+
+/**
+ * Class to check if a Quota has been reached based on the calculation done inside
+ * the Calculator
+ *
+ */
+interface QuotaCalculator<T> {
+ val hasExceededQuota: Boolean
+
+ /**
+ * This function is called to recalculate an old quota score
+ */
+ fun calculateQuota(): Boolean
+
+ /**
+ * Reset the quota progress
+ *
+ * @param newProgress new progress towards the quota
+ */
+ fun resetProgressTowardsQuota(newProgress: T)
+
+ /**
+ * Retrieve the current progress towards the quota
+ *
+ * @return current progress count
+ */
+ fun getProgressTowardsQuota(): T
+}
diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransactionTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransactionTest.kt
index e8c59395e..929b0fcf8 100644
--- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransactionTest.kt
+++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransactionTest.kt
@@ -13,6 +13,7 @@ import io.mockk.mockk
import io.mockk.mockkObject
import io.mockk.unmockkAll
import kotlinx.coroutines.runBlocking
+import org.joda.time.Instant
import org.junit.After
import org.junit.Before
import org.junit.Test
@@ -46,6 +47,10 @@ class RetrieveDiagnosisKeysTransactionTest {
every { LocalData.lastTimeDiagnosisKeysFromServerFetch() } returns Date()
every { LocalData.lastTimeDiagnosisKeysFromServerFetch(any()) } just Runs
every { LocalData.googleApiToken() } returns UUID.randomUUID().toString()
+ every { LocalData.googleAPIProvideDiagnosisKeysCallCount = any() } just Runs
+ every { LocalData.googleAPIProvideDiagnosisKeysCallCount } returns 0
+ every { LocalData.nextTimeRateLimitingUnlocks = any() } just Runs
+ every { LocalData.nextTimeRateLimitingUnlocks } returns Instant.now()
}
@Test
@@ -57,6 +62,7 @@ class RetrieveDiagnosisKeysTransactionTest {
coVerifyOrder {
RetrieveDiagnosisKeysTransaction["executeSetup"]()
+ RetrieveDiagnosisKeysTransaction["executeQuotaCalculation"]()
RetrieveDiagnosisKeysTransaction["executeRetrieveRiskScoreParams"]()
RetrieveDiagnosisKeysTransaction["executeFetchKeyFilesFromServer"](any<Date>())
RetrieveDiagnosisKeysTransaction["executeFetchDateUpdate"](any<Date>())
@@ -77,6 +83,7 @@ class RetrieveDiagnosisKeysTransactionTest {
coVerifyOrder {
RetrieveDiagnosisKeysTransaction["executeSetup"]()
+ RetrieveDiagnosisKeysTransaction["executeQuotaCalculation"]()
RetrieveDiagnosisKeysTransaction["executeRetrieveRiskScoreParams"]()
RetrieveDiagnosisKeysTransaction["executeFetchKeyFilesFromServer"](any<Date>())
RetrieveDiagnosisKeysTransaction["executeAPISubmission"](
diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/CachedKeyFileHolderTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/CachedKeyFileHolderTest.kt
index b03e57dd7..1d6e89794 100644
--- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/CachedKeyFileHolderTest.kt
+++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/CachedKeyFileHolderTest.kt
@@ -38,6 +38,7 @@ class CachedKeyFileHolderTest {
every { KeyCacheRepository.getDateRepository(any()) } returns keyCacheRepository
mockkObject(CachedKeyFileHolder)
coEvery { keyCacheRepository.deleteOutdatedEntries(any()) } just Runs
+ coEvery { keyCacheRepository.clear(any()) } just Runs
}
/**
@@ -53,6 +54,7 @@ class CachedKeyFileHolderTest {
every { CachedKeyFileHolder["checkForFreeSpace"]() } returns Unit
every { CachedKeyFileHolder["getDatesFromServer"]() } returns arrayListOf<String>()
+
runBlocking {
CachedKeyFileHolder.asyncFetchFiles(date)
@@ -63,6 +65,7 @@ class CachedKeyFileHolderTest {
keyCacheRepository.deleteOutdatedEntries(any())
CachedKeyFileHolder["getMissingDaysFromDiff"](arrayListOf<String>())
keyCacheRepository.getDates()
+ keyCacheRepository.clear(emptyList())
keyCacheRepository.getFilesFromEntries()
}
}
diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/GoogleQuotaCalculatorTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/GoogleQuotaCalculatorTest.kt
new file mode 100644
index 000000000..7428be135
--- /dev/null
+++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/GoogleQuotaCalculatorTest.kt
@@ -0,0 +1,295 @@
+package de.rki.coronawarnapp.util
+
+import de.rki.coronawarnapp.storage.LocalData
+import io.mockk.every
+import io.mockk.mockkObject
+import io.mockk.unmockkObject
+import org.joda.time.DateTime
+import org.joda.time.DateTimeUtils
+import org.joda.time.DateTimeZone
+import org.joda.time.Duration
+import org.joda.time.Instant
+import org.joda.time.chrono.GJChronology
+import org.junit.jupiter.api.AfterEach
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.BeforeEach
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.assertThrows
+import testhelpers.BaseTest
+import timber.log.Timber
+import java.util.concurrent.atomic.AtomicInteger
+import java.util.concurrent.atomic.AtomicLong
+
+internal class GoogleQuotaCalculatorTest : BaseTest() {
+
+ private val timeInTest = DateTimeUtils.currentTimeMillis()
+
+ private lateinit var classUnderTest: GoogleQuotaCalculator
+ private val nextTimeRateLimitingUnlocksInTesting = AtomicLong()
+ private val googleAPIProvideDiagnosisKeysCallCount = AtomicInteger()
+
+ private val defaultIncrementByAmountInTest = 14
+ private val defaultQuotaLimitInTest = 20
+
+ @BeforeEach
+ fun setUpClassUnderTest() {
+ classUnderTest = GoogleQuotaCalculator(
+ incrementByAmount = defaultIncrementByAmountInTest,
+ quotaLimit = defaultQuotaLimitInTest,
+ quotaResetPeriod = Duration.standardHours(24),
+ quotaTimeZone = DateTimeZone.UTC,
+ quotaChronology = GJChronology.getInstanceUTC()
+ )
+ DateTimeUtils.setCurrentMillisFixed(timeInTest)
+
+ // Since LocalData is simple to mock
+ mockkObject(LocalData)
+ every { LocalData.nextTimeRateLimitingUnlocks = any() } answers {
+ nextTimeRateLimitingUnlocksInTesting.set((this.arg(0) as Instant).millis)
+ }
+ every { LocalData.nextTimeRateLimitingUnlocks } answers {
+ Instant.ofEpochMilli(nextTimeRateLimitingUnlocksInTesting.get())
+ }
+ every { LocalData.googleAPIProvideDiagnosisKeysCallCount = any() } answers {
+ googleAPIProvideDiagnosisKeysCallCount.set(this.arg(0))
+ }
+ every { LocalData.googleAPIProvideDiagnosisKeysCallCount } answers {
+ googleAPIProvideDiagnosisKeysCallCount.get()
+ }
+
+ }
+
+ @Test
+ fun `isAboveQuota false if called initially`() {
+ assertEquals(classUnderTest.hasExceededQuota, false)
+ }
+
+ @Test
+ fun `isAboveQuota true if called above quota limit when calling with amount bigger than one`() {
+ for (callNumber in 1..5) {
+ classUnderTest.calculateQuota()
+ val aboveQuota = classUnderTest.hasExceededQuota
+ Timber.v("call number $callNumber above quota: $aboveQuota")
+ if (callNumber > 1) {
+ assertEquals(true, aboveQuota)
+ } else {
+ assertEquals(false, aboveQuota)
+ }
+ }
+ }
+
+ @Test
+ fun `getProgressTowardsQuota increases with calls to isAboveQuota but is stopped once increased above the quota`() {
+ var latestCallNumberWithoutLimiting = 1
+ for (callNumber in 1..5) {
+ classUnderTest.calculateQuota()
+ val aboveQuota = classUnderTest.hasExceededQuota
+ Timber.v("call number $callNumber above quota: $aboveQuota")
+ val expectedIncrement = callNumber * defaultIncrementByAmountInTest
+ if (expectedIncrement >= defaultQuotaLimitInTest) {
+ assertEquals(
+ (latestCallNumberWithoutLimiting + 1) * defaultIncrementByAmountInTest,
+ classUnderTest.getProgressTowardsQuota()
+ )
+ } else {
+ assertEquals(
+ callNumber * defaultIncrementByAmountInTest,
+ classUnderTest.getProgressTowardsQuota()
+ )
+ latestCallNumberWithoutLimiting = callNumber
+ }
+ }
+ }
+
+ @Test
+ fun `getProgressTowardsQuota is reset and the quota is not recalculated but isAboveQuota should still be false`() {
+ var latestCallNumberWithoutLimiting = 1
+ for (callNumber in 1..5) {
+ classUnderTest.calculateQuota()
+ val aboveQuota = classUnderTest.hasExceededQuota
+ Timber.v("call number $callNumber above quota: $aboveQuota")
+ val expectedIncrement = callNumber * defaultIncrementByAmountInTest
+ if (expectedIncrement >= defaultQuotaLimitInTest) {
+ assertEquals(
+ (latestCallNumberWithoutLimiting + 1) * defaultIncrementByAmountInTest,
+ classUnderTest.getProgressTowardsQuota()
+ )
+ } else {
+ assertEquals(
+ callNumber * defaultIncrementByAmountInTest,
+ classUnderTest.getProgressTowardsQuota()
+ )
+ latestCallNumberWithoutLimiting = callNumber
+ }
+ }
+
+ classUnderTest.resetProgressTowardsQuota(0)
+ assertEquals(false, classUnderTest.hasExceededQuota)
+ }
+
+ @Test
+ fun `getProgressTowardsQuota is reset but the reset value is no multiple of incrementByAmount`() {
+ var latestCallNumberWithoutLimiting = 1
+ for (callNumber in 1..5) {
+ classUnderTest.calculateQuota()
+ val aboveQuota = classUnderTest.hasExceededQuota
+ Timber.v("call number $callNumber above quota: $aboveQuota")
+ val expectedIncrement = callNumber * defaultIncrementByAmountInTest
+ if (expectedIncrement >= defaultQuotaLimitInTest) {
+ assertEquals(
+ (latestCallNumberWithoutLimiting + 1) * defaultIncrementByAmountInTest,
+ classUnderTest.getProgressTowardsQuota()
+ )
+ } else {
+ assertEquals(
+ callNumber * defaultIncrementByAmountInTest,
+ classUnderTest.getProgressTowardsQuota()
+ )
+ latestCallNumberWithoutLimiting = callNumber
+ }
+ }
+
+ assertThrows<IllegalArgumentException> {
+ classUnderTest.resetProgressTowardsQuota(defaultIncrementByAmountInTest + 1)
+ }
+ }
+
+ @Test
+ fun `getProgressTowardsQuota is reset and the quota is not recalculated and the progress should update`() {
+ var latestCallNumberWithoutLimiting = 1
+ for (callNumber in 1..5) {
+ classUnderTest.calculateQuota()
+ val aboveQuota = classUnderTest.hasExceededQuota
+ Timber.v("call number $callNumber above quota: $aboveQuota")
+ val expectedIncrement = callNumber * defaultIncrementByAmountInTest
+ if (expectedIncrement >= defaultQuotaLimitInTest) {
+ assertEquals(
+ (latestCallNumberWithoutLimiting + 1) * defaultIncrementByAmountInTest,
+ classUnderTest.getProgressTowardsQuota()
+ )
+ } else {
+ assertEquals(
+ callNumber * defaultIncrementByAmountInTest,
+ classUnderTest.getProgressTowardsQuota()
+ )
+ latestCallNumberWithoutLimiting = callNumber
+ }
+ }
+
+ val newProgressAfterReset = 14
+ classUnderTest.resetProgressTowardsQuota(newProgressAfterReset)
+ assertEquals(false, classUnderTest.hasExceededQuota)
+ assertEquals(newProgressAfterReset, classUnderTest.getProgressTowardsQuota())
+ }
+
+ @Test
+ fun `getProgressTowardsQuota is reset and the quota is not recalculated and the progress throws an error because of too high newProgress`() {
+ var latestCallNumberWithoutLimiting = 1
+ var progressBeforeReset: Int? = null
+ for (callNumber in 1..5) {
+ classUnderTest.calculateQuota()
+ val aboveQuota = classUnderTest.hasExceededQuota
+ Timber.v("call number $callNumber above quota: $aboveQuota")
+ val expectedIncrement = callNumber * defaultIncrementByAmountInTest
+ if (expectedIncrement >= defaultQuotaLimitInTest) {
+ progressBeforeReset =
+ (latestCallNumberWithoutLimiting + 1) * defaultIncrementByAmountInTest
+ assertEquals(
+ (latestCallNumberWithoutLimiting + 1) * defaultIncrementByAmountInTest,
+ classUnderTest.getProgressTowardsQuota()
+ )
+ } else {
+ assertEquals(
+ callNumber * defaultIncrementByAmountInTest,
+ classUnderTest.getProgressTowardsQuota()
+ )
+ latestCallNumberWithoutLimiting = callNumber
+ }
+ }
+
+ val newProgressAfterReset = defaultQuotaLimitInTest + 1
+ assertThrows<IllegalArgumentException> {
+ classUnderTest.resetProgressTowardsQuota(newProgressAfterReset)
+ }
+ assertEquals(true, classUnderTest.hasExceededQuota)
+ assertEquals(
+ (progressBeforeReset
+ ?: throw IllegalStateException("progressBeforeReset was not set during test")),
+ classUnderTest.getProgressTowardsQuota()
+ )
+ }
+
+ @Test
+ fun `isAboveQuota true if called above quota limit when calling with amount one`() {
+ classUnderTest = GoogleQuotaCalculator(
+ incrementByAmount = 1,
+ quotaLimit = 3,
+ quotaResetPeriod = Duration.standardHours(24),
+ quotaTimeZone = DateTimeZone.UTC,
+ quotaChronology = GJChronology.getInstanceUTC()
+ )
+ for (callNumber in 1..15) {
+ classUnderTest.calculateQuota()
+ val aboveQuota = classUnderTest.hasExceededQuota
+ Timber.v("call number $callNumber above quota: $aboveQuota")
+ if (callNumber > 3) {
+ assertEquals(true, aboveQuota)
+ } else {
+ assertEquals(false, aboveQuota)
+ }
+ }
+ }
+
+ @Test
+ fun `isAboveQuota false if called above quota limit but next day resets quota`() {
+ for (callNumber in 1..5) {
+ classUnderTest.calculateQuota()
+ val aboveQuota = classUnderTest.hasExceededQuota
+ Timber.v("call number $callNumber above quota: $aboveQuota")
+ if (callNumber > 1) {
+ assertEquals(true, aboveQuota)
+ } else {
+ assertEquals(false, aboveQuota)
+ }
+ }
+
+ // Day Change
+ val timeInTestAdvancedByADay = timeInTest + Duration.standardDays(1).millis
+ DateTimeUtils.setCurrentMillisFixed(timeInTestAdvancedByADay)
+ classUnderTest.calculateQuota()
+ val aboveQuotaAfterDayAdvance = classUnderTest.hasExceededQuota
+ Timber.v("above quota after day advance: $aboveQuotaAfterDayAdvance")
+
+ assertEquals(false, aboveQuotaAfterDayAdvance)
+ }
+
+ @Test
+ fun `test if isAfter is affected by Timezone to make sure we do not run into Shifting Errors`() {
+ val testTimeUTC = DateTime(
+ timeInTest,
+ DateTimeZone.UTC
+ ).withChronology(GJChronology.getInstanceUTC())
+ val testTimeGMT = DateTime(
+ timeInTest,
+ DateTimeZone.forID("Etc/GMT+2")
+ ).withChronology(GJChronology.getInstanceUTC())
+
+ assertEquals(testTimeGMT, testTimeUTC)
+ assertEquals(testTimeGMT.millis, testTimeUTC.millis)
+
+ val testTimeUTCAfterGMT = testTimeUTC.plusMinutes(1)
+
+ assertEquals(true, testTimeUTCAfterGMT.isAfter(testTimeGMT))
+
+ val testTimeGMTAfterUTC = testTimeGMT.plusMinutes(1)
+
+ assertEquals(true, testTimeGMTAfterUTC.isAfter(testTimeUTC))
+
+ }
+
+ @AfterEach
+ fun cleanup() {
+ DateTimeUtils.setCurrentMillisSystem()
+ unmockkObject(LocalData)
+ }
+}
\ No newline at end of file
--
GitLab