-
Notifications
You must be signed in to change notification settings - Fork 87
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
EFGS downloaded keys upsert through insert manager
- Loading branch information
Showing
17 changed files
with
435 additions
and
34 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
89 changes: 89 additions & 0 deletions
89
...ops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/InteropsInsertManager.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| package org.dpppt.backend.sdk.interops.insertmanager; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.Map.Entry; | ||
| import java.util.stream.Collectors; | ||
| import org.dpppt.backend.sdk.data.gaen.GaenDataService; | ||
| import org.dpppt.backend.sdk.interops.insertmanager.insertionfilters.InteropsKeyInsertionFilter; | ||
| import org.dpppt.backend.sdk.interops.insertmanager.insertionmodifier.InteropsKeyInsertionModifier; | ||
| import org.dpppt.backend.sdk.model.gaen.GaenKeyForInterops; | ||
| import org.dpppt.backend.sdk.utils.UTCInstant; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| /** | ||
| * The insertion manager is responsible for inserting keys downloaded from an international | ||
| * gateway/hub into the database. To make sure we only have valid keys in the database, a list of | ||
| * {@link InteropsKeyInsertionModifier} is applied, and then a list of {@link | ||
| * InteropsKeyInsertionFilter} is applied to the given list of keys. The remaining keys are then | ||
| * inserted into the database. | ||
| */ | ||
| public class InteropsInsertManager { | ||
|
|
||
| private final List<InteropsKeyInsertionFilter> filterList = new ArrayList<>(); | ||
| private final List<InteropsKeyInsertionModifier> modifierList = new ArrayList<>(); | ||
|
|
||
| private final GaenDataService dataService; | ||
|
|
||
| private static final Logger logger = LoggerFactory.getLogger(InteropsInsertManager.class); | ||
|
|
||
| public InteropsInsertManager(GaenDataService dataService) { | ||
| this.dataService = dataService; | ||
| } | ||
|
|
||
| public void addFilter(InteropsKeyInsertionFilter filter) { | ||
| this.filterList.add(filter); | ||
| } | ||
|
|
||
| public void addModifier(InteropsKeyInsertionModifier modifier) { | ||
| this.modifierList.add(modifier); | ||
| } | ||
|
|
||
| /** | ||
| * Inserts the keys into the database. The additional parameters are supplied to the configured | ||
| * modifiers and filters. | ||
| * | ||
| * @param keys the list of downloaded international keys | ||
| * @param now current timestamp to work with. | ||
| * @param batchTag | ||
| */ | ||
| public void insertIntoDatabase(List<GaenKeyForInterops> keys, UTCInstant now, String batchTag) { | ||
| if (keys == null || keys.isEmpty()) { | ||
| return; | ||
| } | ||
| var internalKeys = modifyAndFilter(keys, now); | ||
| if (!internalKeys.isEmpty()) { | ||
| for (Entry<String, List<GaenKeyForInterops>> keysForOrigin : | ||
| internalKeys.stream() | ||
| .collect(Collectors.groupingBy(GaenKeyForInterops::getOrigin)) | ||
| .entrySet()) { | ||
| dataService.upsertExposeeFromInterops( | ||
| keysForOrigin.getValue().stream() | ||
| .map(GaenKeyForInterops::getGaenKey) | ||
| .collect(Collectors.toList()), | ||
| now, | ||
| keysForOrigin.getKey(), | ||
| batchTag); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private List<GaenKeyForInterops> modifyAndFilter(List<GaenKeyForInterops> keys, UTCInstant now) { | ||
| var internalKeys = keys; | ||
|
|
||
| for (InteropsKeyInsertionModifier modifier : modifierList) { | ||
| internalKeys = modifier.modify(now, internalKeys); | ||
| } | ||
|
|
||
| for (InteropsKeyInsertionFilter filter : filterList) { | ||
| int sizeBefore = internalKeys.size(); | ||
| internalKeys = filter.filter(now, internalKeys); | ||
| logger.info( | ||
| "{} keys filtered out by {} filter", | ||
| (internalKeys.size() - sizeBefore), | ||
| filter.getName()); | ||
| } | ||
| return internalKeys; | ||
| } | ||
| } |
38 changes: 38 additions & 0 deletions
38
...n/java/org/dpppt/backend/sdk/interops/insertmanager/insertionfilters/AssertKeyFormat.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| package org.dpppt.backend.sdk.interops.insertmanager.insertionfilters; | ||
|
|
||
| import java.util.Base64; | ||
| import java.util.List; | ||
| import java.util.stream.Collectors; | ||
| import org.dpppt.backend.sdk.model.gaen.GaenKeyForInterops; | ||
| import org.dpppt.backend.sdk.utils.UTCInstant; | ||
|
|
||
| /** Filters out keys with invalid base64 encoding or incorrect length. */ | ||
| public class AssertKeyFormat implements InteropsKeyInsertionFilter { | ||
|
|
||
| private final int gaenKeySizeBytes; | ||
|
|
||
| public AssertKeyFormat(int gaenKeySizeBytes) { | ||
| this.gaenKeySizeBytes = gaenKeySizeBytes; | ||
| } | ||
|
|
||
| @Override | ||
| public List<GaenKeyForInterops> filter(UTCInstant now, List<GaenKeyForInterops> content) { | ||
| return content.stream() | ||
| .filter(key -> isValidKeyFormat(key.getKeyData())) | ||
| .collect(Collectors.toList()); | ||
| } | ||
|
|
||
| private boolean isValidKeyFormat(String value) { | ||
| try { | ||
| byte[] key = Base64.getDecoder().decode(value); | ||
| return key.length == gaenKeySizeBytes; | ||
| } catch (Exception e) { | ||
| return false; | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public String getName() { | ||
| return "AssertKeyFormat"; | ||
| } | ||
| } |
47 changes: 47 additions & 0 deletions
47
...c/main/java/org/dpppt/backend/sdk/interops/insertmanager/insertionfilters/DsosFilter.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| package org.dpppt.backend.sdk.interops.insertmanager.insertionfilters; | ||
|
|
||
| import java.util.List; | ||
| import java.util.stream.Collectors; | ||
| import org.dpppt.backend.sdk.model.gaen.GaenKeyForInterops; | ||
| import org.dpppt.backend.sdk.utils.UTCInstant; | ||
|
|
||
| /** | ||
| * Filter keys that have a day since onset of symptoms (dsos) not relevant for our epidemiological | ||
| * parameters | ||
| */ | ||
| public class DsosFilter implements InteropsKeyInsertionFilter { | ||
|
|
||
| @Override | ||
| public List<GaenKeyForInterops> filter(UTCInstant now, List<GaenKeyForInterops> content) { | ||
| return content.stream().filter(key -> !hasIrrelevantDsos(key)).collect(Collectors.toList()); | ||
| } | ||
|
|
||
| private boolean hasIrrelevantDsos(GaenKeyForInterops key) { | ||
| Integer dsos = key.getDaysSinceOnsetOfSymptoms(); | ||
| if (dsos == null) { // keys with dsos null values are dropped | ||
| return true; | ||
| } | ||
| int normalizedDsos = dsos; | ||
| if (dsos < 20) { // onset known. dsos in [-14, +14] | ||
| // dsos is already normalized | ||
| } else if (dsos < 1986) { // onset range `n` days (n < 19). dsos in [n*100-14, n*100+14]. | ||
| final int nMax = 19; | ||
| final int n = dsos + nMax / 100; | ||
| int endOfRange = n * 100; | ||
| int startOfRange = endOfRange - n; | ||
| normalizedDsos = dsos - startOfRange; | ||
| } else if (dsos < 2986) { // unknown onset. dsos in [1986, 2014] | ||
| normalizedDsos = dsos - 2000; | ||
| } else if (dsos < 3986) { // asymptomatic. dsos in [2986, 3014] | ||
| normalizedDsos = dsos - 3000; | ||
| } else { // unknown symptom status. dsos in [3986, 4014] | ||
| normalizedDsos = dsos - 4000; | ||
| } | ||
| return normalizedDsos < -2; | ||
| } | ||
|
|
||
| @Override | ||
| public String getName() { | ||
| return "DSOS"; | ||
| } | ||
| } |
37 changes: 37 additions & 0 deletions
37
...org/dpppt/backend/sdk/interops/insertmanager/insertionfilters/EnforceRetentionPeriod.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| package org.dpppt.backend.sdk.interops.insertmanager.insertionfilters; | ||
|
|
||
| import java.time.Duration; | ||
| import java.util.List; | ||
| import java.util.stream.Collectors; | ||
| import org.dpppt.backend.sdk.model.gaen.GaenKeyForInterops; | ||
| import org.dpppt.backend.sdk.model.gaen.GaenUnit; | ||
| import org.dpppt.backend.sdk.utils.UTCInstant; | ||
|
|
||
| /** | ||
| * Checks if a key is in the configured retention period. If a key is before the retention period it | ||
| * is filtered out, as it will not be relevant for the system anymore. | ||
| */ | ||
| public class EnforceRetentionPeriod implements InteropsKeyInsertionFilter { | ||
|
|
||
| private final Duration retentionPeriod; | ||
|
|
||
| public EnforceRetentionPeriod(Duration retentionPeriod) { | ||
| this.retentionPeriod = retentionPeriod; | ||
| } | ||
|
|
||
| @Override | ||
| public List<GaenKeyForInterops> filter(UTCInstant now, List<GaenKeyForInterops> content) { | ||
| return content.stream() | ||
| .filter( | ||
| key -> { | ||
| var timestamp = UTCInstant.of(key.getRollingStartNumber(), GaenUnit.TenMinutes); | ||
| return !timestamp.isBeforeDateOf(now.minus(retentionPeriod)); | ||
| }) | ||
| .collect(Collectors.toList()); | ||
| } | ||
|
|
||
| @Override | ||
| public String getName() { | ||
| return "EnforeRetentionPeriod"; | ||
| } | ||
| } |
27 changes: 27 additions & 0 deletions
27
.../dpppt/backend/sdk/interops/insertmanager/insertionfilters/EnforceValidRollingPeriod.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| package org.dpppt.backend.sdk.interops.insertmanager.insertionfilters; | ||
|
|
||
| import java.util.List; | ||
| import java.util.stream.Collectors; | ||
| import org.dpppt.backend.sdk.model.gaen.GaenKeyForInterops; | ||
| import org.dpppt.backend.sdk.utils.UTCInstant; | ||
|
|
||
| /** | ||
| * This filter checks for valid rolling period. The rolling period must always be in [1..144], | ||
| * otherwise the key is not valid and is filtered out. See <a href= | ||
| * "https://github.com/google/exposure-notifications-server/blob/main/docs/server_functional_requirements.md#publishing-temporary-exposure-keys" | ||
| * >EN documentation</a> | ||
| */ | ||
| public class EnforceValidRollingPeriod implements InteropsKeyInsertionFilter { | ||
|
|
||
| @Override | ||
| public List<GaenKeyForInterops> filter(UTCInstant now, List<GaenKeyForInterops> content) { | ||
| return content.stream() | ||
| .filter(key -> key.getRollingPeriod() >= 1 && key.getRollingPeriod() <= 144) | ||
| .collect(Collectors.toList()); | ||
| } | ||
|
|
||
| @Override | ||
| public String getName() { | ||
| return "EnforeValidRollingPeriod"; | ||
| } | ||
| } |
Oops, something went wrong.