diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-data/src/main/java/org/dpppt/backend/sdk/data/gaen/GaenDataService.java b/dpppt-backend-sdk/dpppt-backend-sdk-data/src/main/java/org/dpppt/backend/sdk/data/gaen/GaenDataService.java index d7c24f43..39042d2a 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-data/src/main/java/org/dpppt/backend/sdk/data/gaen/GaenDataService.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-data/src/main/java/org/dpppt/backend/sdk/data/gaen/GaenDataService.java @@ -19,13 +19,15 @@ public interface GaenDataService { /** - * Upserts (Update or Inserts) the given key received from interops synchronization. + * Upserts (Update or Inserts) the given keys received from interops synchronization. * - * @param key the exposed key to upsert + * @param keys the exposed keys to upsert * @param now time of the sync * @param origin the origin or the key + * @param batchTag batchTag of downloaded key */ - void upsertExposeeFromInterops(GaenKey key, UTCInstant now, String origin); + void upsertExposeeFromInterops( + List keys, UTCInstant now, String origin, String batchTag); /** * Upserts (Update or Inserts) the given list of exposed keys diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-data/src/main/java/org/dpppt/backend/sdk/data/gaen/JdbcGaenDataServiceImpl.java b/dpppt-backend-sdk/dpppt-backend-sdk-data/src/main/java/org/dpppt/backend/sdk/data/gaen/JdbcGaenDataServiceImpl.java index 1492a911..0070fda2 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-data/src/main/java/org/dpppt/backend/sdk/data/gaen/JdbcGaenDataServiceImpl.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-data/src/main/java/org/dpppt/backend/sdk/data/gaen/JdbcGaenDataServiceImpl.java @@ -59,8 +59,11 @@ public JdbcGaenDataServiceImpl( @Override @Transactional(readOnly = false) - public void upsertExposeeFromInterops(GaenKey gaenKey, UTCInstant now, String origin) { - internalUpsertKey(gaenKey, now, origin, true); + public void upsertExposeeFromInterops( + List gaenKeys, UTCInstant now, String origin, String batchTag) { + for (GaenKey gaenKey : gaenKeys) { + internalUpsertKey(gaenKey, now, origin, batchTag, true); + } } @Override @@ -84,7 +87,7 @@ public void upsertExposeesDelayed( : delayedReceivedAt; for (var gaenKey : gaenKeys) { - internalUpsertKey(gaenKey, receivedAt, this.originCountry, withFederationGateway); + internalUpsertKey(gaenKey, receivedAt, this.originCountry, null, withFederationGateway); } } @@ -294,25 +297,30 @@ public void cleanDB(Duration retentionPeriod) { } private void internalUpsertKey( - GaenKey gaenKey, UTCInstant receivedAt, String origin, boolean withFederationGateway) { + GaenKey gaenKey, + UTCInstant receivedAt, + String origin, + String batchTag, + boolean withFederationGateway) { String sqlKey = null; if (dbType.equals(PGSQL)) { sqlKey = "insert into t_gaen_exposed (key, rolling_start_number, rolling_period, received_at," - + " origin, share_with_federation_gateway) values (:key, :rolling_start_number," - + " :rolling_period, :received_at, :origin, :share_with_federation_gateway) on" + + " origin, share_with_federation_gateway, batch_tag)" + + " values (:key, :rolling_start_number, :rolling_period, :received_at," + + " :origin, :share_with_federation_gateway, :batch_tag) on" + " conflict on constraint gaen_exposed_key do nothing"; } else { sqlKey = "merge into t_gaen_exposed using (values(cast(:key as varchar(24))," + " :rolling_start_number, :rolling_period, :received_at, cast(:origin as" - + " varchar(10)), :share_with_federation_gateway)) as vals(key," - + " rolling_start_number, rolling_period, received_at, origin," - + " share_with_federation_gateway) on t_gaen_exposed.key = vals.key when not matched" - + " then insert (key, rolling_start_number, rolling_period, received_at, origin," - + " share_with_federation_gateway) values (vals.key, vals.rolling_start_number," - + " vals.rolling_period, vals.received_at, vals.origin," - + " vals.share_with_federation_gateway)"; + + " varchar(10)), :share_with_federation_gateway, cast(:batch_tag as varchar(50))))" + + " as vals(key, rolling_start_number, rolling_period, received_at, origin," + + " share_with_federation_gateway, batch_tag) on t_gaen_exposed.key = vals.key" + + " when not matched then insert (key, rolling_start_number, rolling_period," + + " received_at, origin, share_with_federation_gateway, batch_tag)" + + " values (vals.key, vals.rolling_start_number, vals.rolling_period," + + " vals.received_at, vals.origin, vals.share_with_federation_gateway, vals.batch_tag)"; } MapSqlParameterSource params = new MapSqlParameterSource(); @@ -322,6 +330,7 @@ private void internalUpsertKey( params.addValue("received_at", receivedAt.getDate()); params.addValue("origin", origin); params.addValue("share_with_federation_gateway", withFederationGateway); + params.addValue("batch_tag", batchTag); KeyHolder keyHolder = new GeneratedKeyHolder(); jt.update(sqlKey, params, keyHolder); } diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/config/WSBaseConfig.java b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/config/WSBaseConfig.java index 677636c9..41887c02 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/config/WSBaseConfig.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/config/WSBaseConfig.java @@ -17,6 +17,12 @@ import org.dpppt.backend.sdk.data.gaen.JdbcGaenDataServiceImpl; import org.dpppt.backend.sdk.data.interops.JdbcSyncLogDataServiceImpl; import org.dpppt.backend.sdk.data.interops.SyncLogDataService; +import org.dpppt.backend.sdk.interops.insertmanager.InteropsInsertManager; +import org.dpppt.backend.sdk.interops.insertmanager.insertionfilters.AssertKeyFormat; +import org.dpppt.backend.sdk.interops.insertmanager.insertionfilters.DsosFilter; +import org.dpppt.backend.sdk.interops.insertmanager.insertionfilters.EnforceRetentionPeriod; +import org.dpppt.backend.sdk.interops.insertmanager.insertionfilters.EnforceValidRollingPeriod; +import org.dpppt.backend.sdk.interops.insertmanager.insertionfilters.RemoveKeysFromFuture; import org.dpppt.backend.sdk.interops.model.HubConfigs; import org.dpppt.backend.sdk.interops.syncer.EfgsHubSyncer; import org.dpppt.backend.sdk.interops.syncer.efgs.EfgsClient; @@ -37,6 +43,9 @@ public abstract class WSBaseConfig implements WebMvcConfigurer { @Value("${ws.exposedlist.releaseBucketDuration: 7200000}") long releaseBucketDuration; + @Value("${ws.app.gaen.key_size: 16}") + int gaenKeySizeBytes; + @Value("${ws.app.gaen.timeskew:PT2h}") Duration timeSkew; @@ -71,8 +80,24 @@ public EfgsClient efgsClient(HubConfigs hubConfigs) throws CertificateException public EfgsHubSyncer efgsHubSyncer( EfgsClient efgsClient, GaenDataService gaenDataService, - SyncLogDataService syncLogDataService) { + SyncLogDataService syncLogDataService, + InteropsInsertManager interopsInsertManager) { return new EfgsHubSyncer( - efgsClient, Duration.ofDays(retentionDays), gaenDataService, syncLogDataService); + efgsClient, + Duration.ofDays(retentionDays), + gaenDataService, + syncLogDataService, + interopsInsertManager); + } + + @Bean + public InteropsInsertManager interopsInsertManager(GaenDataService gaenDataService) { + var manager = new InteropsInsertManager(gaenDataService); + manager.addFilter(new AssertKeyFormat(gaenKeySizeBytes)); + manager.addFilter(new RemoveKeysFromFuture()); + manager.addFilter(new EnforceRetentionPeriod(Duration.ofDays(retentionDays))); + manager.addFilter(new EnforceValidRollingPeriod()); + manager.addFilter(new DsosFilter()); + return manager; } } diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/InteropsInsertManager.java b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/InteropsInsertManager.java new file mode 100644 index 00000000..3459c5c8 --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/InteropsInsertManager.java @@ -0,0 +1,89 @@ +package org.dpppt.backend.sdk.interops.insertmanager; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map.Entry; +import java.util.stream.Collectors; +import org.dpppt.backend.sdk.data.gaen.GaenDataService; +import org.dpppt.backend.sdk.interops.insertmanager.insertionfilters.InteropsKeyInsertionFilter; +import org.dpppt.backend.sdk.interops.insertmanager.insertionmodifier.InteropsKeyInsertionModifier; +import org.dpppt.backend.sdk.model.gaen.GaenKeyForInterops; +import org.dpppt.backend.sdk.utils.UTCInstant; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The insertion manager is responsible for inserting keys downloaded from an international + * gateway/hub into the database. To make sure we only have valid keys in the database, a list of + * {@link InteropsKeyInsertionModifier} is applied, and then a list of {@link + * InteropsKeyInsertionFilter} is applied to the given list of keys. The remaining keys are then + * inserted into the database. + */ +public class InteropsInsertManager { + + private final List filterList = new ArrayList<>(); + private final List modifierList = new ArrayList<>(); + + private final GaenDataService dataService; + + private static final Logger logger = LoggerFactory.getLogger(InteropsInsertManager.class); + + public InteropsInsertManager(GaenDataService dataService) { + this.dataService = dataService; + } + + public void addFilter(InteropsKeyInsertionFilter filter) { + this.filterList.add(filter); + } + + public void addModifier(InteropsKeyInsertionModifier modifier) { + this.modifierList.add(modifier); + } + + /** + * Inserts the keys into the database. The additional parameters are supplied to the configured + * modifiers and filters. + * + * @param keys the list of downloaded international keys + * @param now current timestamp to work with. + * @param batchTag + */ + public void insertIntoDatabase(List keys, UTCInstant now, String batchTag) { + if (keys == null || keys.isEmpty()) { + return; + } + var internalKeys = modifyAndFilter(keys, now); + if (!internalKeys.isEmpty()) { + for (Entry> keysForOrigin : + internalKeys.stream() + .collect(Collectors.groupingBy(GaenKeyForInterops::getOrigin)) + .entrySet()) { + dataService.upsertExposeeFromInterops( + keysForOrigin.getValue().stream() + .map(GaenKeyForInterops::getGaenKey) + .collect(Collectors.toList()), + now, + keysForOrigin.getKey(), + batchTag); + } + } + } + + private List modifyAndFilter(List keys, UTCInstant now) { + var internalKeys = keys; + + for (InteropsKeyInsertionModifier modifier : modifierList) { + internalKeys = modifier.modify(now, internalKeys); + } + + for (InteropsKeyInsertionFilter filter : filterList) { + int sizeBefore = internalKeys.size(); + internalKeys = filter.filter(now, internalKeys); + logger.info( + "{} keys filtered out by {} filter", + (internalKeys.size() - sizeBefore), + filter.getName()); + } + return internalKeys; + } +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/insertionfilters/AssertKeyFormat.java b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/insertionfilters/AssertKeyFormat.java new file mode 100644 index 00000000..f06dd4a5 --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/insertionfilters/AssertKeyFormat.java @@ -0,0 +1,38 @@ +package org.dpppt.backend.sdk.interops.insertmanager.insertionfilters; + +import java.util.Base64; +import java.util.List; +import java.util.stream.Collectors; +import org.dpppt.backend.sdk.model.gaen.GaenKeyForInterops; +import org.dpppt.backend.sdk.utils.UTCInstant; + +/** Filters out keys with invalid base64 encoding or incorrect length. */ +public class AssertKeyFormat implements InteropsKeyInsertionFilter { + + private final int gaenKeySizeBytes; + + public AssertKeyFormat(int gaenKeySizeBytes) { + this.gaenKeySizeBytes = gaenKeySizeBytes; + } + + @Override + public List filter(UTCInstant now, List content) { + return content.stream() + .filter(key -> isValidKeyFormat(key.getKeyData())) + .collect(Collectors.toList()); + } + + private boolean isValidKeyFormat(String value) { + try { + byte[] key = Base64.getDecoder().decode(value); + return key.length == gaenKeySizeBytes; + } catch (Exception e) { + return false; + } + } + + @Override + public String getName() { + return "AssertKeyFormat"; + } +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/insertionfilters/DsosFilter.java b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/insertionfilters/DsosFilter.java new file mode 100644 index 00000000..dc52a41c --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/insertionfilters/DsosFilter.java @@ -0,0 +1,47 @@ +package org.dpppt.backend.sdk.interops.insertmanager.insertionfilters; + +import java.util.List; +import java.util.stream.Collectors; +import org.dpppt.backend.sdk.model.gaen.GaenKeyForInterops; +import org.dpppt.backend.sdk.utils.UTCInstant; + +/** + * Filter keys that have a day since onset of symptoms (dsos) not relevant for our epidemiological + * parameters + */ +public class DsosFilter implements InteropsKeyInsertionFilter { + + @Override + public List filter(UTCInstant now, List content) { + return content.stream().filter(key -> !hasIrrelevantDsos(key)).collect(Collectors.toList()); + } + + private boolean hasIrrelevantDsos(GaenKeyForInterops key) { + Integer dsos = key.getDaysSinceOnsetOfSymptoms(); + if (dsos == null) { // keys with dsos null values are dropped + return true; + } + int normalizedDsos = dsos; + if (dsos < 20) { // onset known. dsos in [-14, +14] + // dsos is already normalized + } else if (dsos < 1986) { // onset range `n` days (n < 19). dsos in [n*100-14, n*100+14]. + final int nMax = 19; + final int n = dsos + nMax / 100; + int endOfRange = n * 100; + int startOfRange = endOfRange - n; + normalizedDsos = dsos - startOfRange; + } else if (dsos < 2986) { // unknown onset. dsos in [1986, 2014] + normalizedDsos = dsos - 2000; + } else if (dsos < 3986) { // asymptomatic. dsos in [2986, 3014] + normalizedDsos = dsos - 3000; + } else { // unknown symptom status. dsos in [3986, 4014] + normalizedDsos = dsos - 4000; + } + return normalizedDsos < -2; + } + + @Override + public String getName() { + return "DSOS"; + } +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/insertionfilters/EnforceRetentionPeriod.java b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/insertionfilters/EnforceRetentionPeriod.java new file mode 100644 index 00000000..7192987b --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/insertionfilters/EnforceRetentionPeriod.java @@ -0,0 +1,37 @@ +package org.dpppt.backend.sdk.interops.insertmanager.insertionfilters; + +import java.time.Duration; +import java.util.List; +import java.util.stream.Collectors; +import org.dpppt.backend.sdk.model.gaen.GaenKeyForInterops; +import org.dpppt.backend.sdk.model.gaen.GaenUnit; +import org.dpppt.backend.sdk.utils.UTCInstant; + +/** + * Checks if a key is in the configured retention period. If a key is before the retention period it + * is filtered out, as it will not be relevant for the system anymore. + */ +public class EnforceRetentionPeriod implements InteropsKeyInsertionFilter { + + private final Duration retentionPeriod; + + public EnforceRetentionPeriod(Duration retentionPeriod) { + this.retentionPeriod = retentionPeriod; + } + + @Override + public List filter(UTCInstant now, List content) { + return content.stream() + .filter( + key -> { + var timestamp = UTCInstant.of(key.getRollingStartNumber(), GaenUnit.TenMinutes); + return !timestamp.isBeforeDateOf(now.minus(retentionPeriod)); + }) + .collect(Collectors.toList()); + } + + @Override + public String getName() { + return "EnforeRetentionPeriod"; + } +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/insertionfilters/EnforceValidRollingPeriod.java b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/insertionfilters/EnforceValidRollingPeriod.java new file mode 100644 index 00000000..fb3cfc3f --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/insertionfilters/EnforceValidRollingPeriod.java @@ -0,0 +1,27 @@ +package org.dpppt.backend.sdk.interops.insertmanager.insertionfilters; + +import java.util.List; +import java.util.stream.Collectors; +import org.dpppt.backend.sdk.model.gaen.GaenKeyForInterops; +import org.dpppt.backend.sdk.utils.UTCInstant; + +/** + * This filter checks for valid rolling period. The rolling period must always be in [1..144], + * otherwise the key is not valid and is filtered out. See EN documentation + */ +public class EnforceValidRollingPeriod implements InteropsKeyInsertionFilter { + + @Override + public List filter(UTCInstant now, List content) { + return content.stream() + .filter(key -> key.getRollingPeriod() >= 1 && key.getRollingPeriod() <= 144) + .collect(Collectors.toList()); + } + + @Override + public String getName() { + return "EnforeValidRollingPeriod"; + } +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/insertionfilters/InteropsKeyInsertionFilter.java b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/insertionfilters/InteropsKeyInsertionFilter.java new file mode 100644 index 00000000..971b1913 --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/insertionfilters/InteropsKeyInsertionFilter.java @@ -0,0 +1,23 @@ +package org.dpppt.backend.sdk.interops.insertmanager.insertionfilters; + +import java.util.List; +import org.dpppt.backend.sdk.interops.insertmanager.InteropsInsertManager; +import org.dpppt.backend.sdk.model.gaen.GaenKeyForInterops; +import org.dpppt.backend.sdk.utils.UTCInstant; + +/** Interface for filters than can be configured in the {@link InteropsInsertManager} */ +public interface InteropsKeyInsertionFilter { + + /** + * The {@link InteropsInsertManager} goes through all configured filters and calls them with a + * list of {@link GaenKeyForInterops} where the filters are applied before inserting into the + * database. + * + * @param now current timestamp + * @param content the list of new gaen keys for insertion + * @return + */ + public List filter(UTCInstant now, List content); + + public String getName(); +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/insertionfilters/RemoveKeysFromFuture.java b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/insertionfilters/RemoveKeysFromFuture.java new file mode 100644 index 00000000..bef9338c --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/insertionfilters/RemoveKeysFromFuture.java @@ -0,0 +1,30 @@ +package org.dpppt.backend.sdk.interops.insertmanager.insertionfilters; + +import java.util.List; +import java.util.stream.Collectors; +import org.dpppt.backend.sdk.model.gaen.GaenKeyForInterops; +import org.dpppt.backend.sdk.model.gaen.GaenUnit; +import org.dpppt.backend.sdk.utils.UTCInstant; + +/** + * Reject keys that are too far in the future. The `rollingStart` must not be later than tomorrow. + */ +public class RemoveKeysFromFuture implements InteropsKeyInsertionFilter { + + @Override + public List filter(UTCInstant now, List content) { + return content.stream() + .filter( + key -> { + var rollingStartNumberInstant = + UTCInstant.of(key.getRollingStartNumber(), GaenUnit.TenMinutes); + return rollingStartNumberInstant.isBeforeDateOf(now.plusDays(2)); + }) + .collect(Collectors.toList()); + } + + @Override + public String getName() { + return "RemoveKeysFromFuture"; + } +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/insertionmodifier/InteropsKeyInsertionModifier.java b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/insertionmodifier/InteropsKeyInsertionModifier.java new file mode 100644 index 00000000..11f9e4ed --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/insertmanager/insertionmodifier/InteropsKeyInsertionModifier.java @@ -0,0 +1,21 @@ +package org.dpppt.backend.sdk.interops.insertmanager.insertionmodifier; + +import java.util.List; +import org.dpppt.backend.sdk.interops.insertmanager.InteropsInsertManager; +import org.dpppt.backend.sdk.model.gaen.GaenKeyForInterops; +import org.dpppt.backend.sdk.utils.UTCInstant; + +/** Interface for key modifiers than can be configured in the {@link InteropsInsertManager} */ +public interface InteropsKeyInsertionModifier { + + /** + * The {@link InteropsInsertManager} goes through all configured key modifiers and calls them with + * a list of {@link GaenKeyForInterops} where the modifieres are applied before inserting into the + * database. + * + * @param now current timestamp + * @param content the list of new gaen keys for modification + * @return + */ + public List modify(UTCInstant now, List content); +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/syncer/EfgsHubSyncer.java b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/syncer/EfgsHubSyncer.java index 022b0b11..f6ad5627 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/syncer/EfgsHubSyncer.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/syncer/EfgsHubSyncer.java @@ -20,6 +20,7 @@ import org.apache.commons.codec.binary.Base64; import org.dpppt.backend.sdk.data.gaen.GaenDataService; import org.dpppt.backend.sdk.data.interops.SyncLogDataService; +import org.dpppt.backend.sdk.interops.insertmanager.InteropsInsertManager; import org.dpppt.backend.sdk.interops.model.GaenKeyBatch; import org.dpppt.backend.sdk.interops.syncer.efgs.EfgsClient; import org.dpppt.backend.sdk.model.gaen.GaenKeyForInterops; @@ -47,16 +48,19 @@ public class EfgsHubSyncer { private final Duration retentionPeriod; private final GaenDataService gaenDataService; private final SyncLogDataService syncLogDataService; + private final InteropsInsertManager insertManager; public EfgsHubSyncer( EfgsClient efgsClient, Duration retentionPeriod, GaenDataService gaenDataService, - SyncLogDataService syncLogDataService) { + SyncLogDataService syncLogDataService, + InteropsInsertManager interopsInsertManager) { this.efgsClient = efgsClient; this.retentionPeriod = retentionPeriod; this.gaenDataService = gaenDataService; this.syncLogDataService = syncLogDataService; + this.insertManager = interopsInsertManager; } public void sync() { @@ -122,7 +126,8 @@ public void download(LocalDate today) { dayComplete = keyBatch.isLastBatchForDay(); if (!keyBatch.getKeys().isEmpty()) { logger.info("upserting downloaded batch: {}", keyBatch); - upsertKeys(keyBatch.getKeys()); + insertManager.insertIntoDatabase( + keyBatch.getKeys(), UTCInstant.now(), downloadedBatchTag); logDownload(start, date, downloadedBatchTag, true); } else if (downloadedBatchTag != null) { logger.info("empty batch: {}", downloadedBatchTag); @@ -139,13 +144,6 @@ public void download(LocalDate today) { logger.info("Download done"); } - private void upsertKeys(List keys) { // TODO insert manager - UTCInstant now = UTCInstant.now(); - for (GaenKeyForInterops key : keys) { - gaenDataService.upsertExposeeFromInterops(key.getGaenKey(), now, key.getOrigin()); - } - } - private String generateBatchTag(int counter, byte[] runnerHash) { var now = LocalDateTime.now(ZoneOffset.UTC); return String.format( diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/syncer/IrishHubSyncer.java b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/syncer/IrishHubSyncer.java index e01ea36e..bf9ced1d 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/syncer/IrishHubSyncer.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/main/java/org/dpppt/backend/sdk/interops/syncer/IrishHubSyncer.java @@ -229,7 +229,8 @@ private void download(LocalDate today) throws URISyntaxException { if (irishKey.getOrigin() != null && !irishKey.getOrigin().isBlank() && !irishKey.getRegions().isEmpty()) { - gaenDataService.upsertExposeeFromInterops(gaenKey, now, irishKey.getOrigin()); + gaenDataService.upsertExposeeFromInterops( + List.of(gaenKey), now, irishKey.getOrigin(), null); } } } diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/test/java/org/dpppt/backend/sdk/interops/SyncTest.java b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/test/java/org/dpppt/backend/sdk/interops/SyncTest.java index 1d424411..dc681ad7 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/test/java/org/dpppt/backend/sdk/interops/SyncTest.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/test/java/org/dpppt/backend/sdk/interops/SyncTest.java @@ -15,8 +15,10 @@ import org.dpppt.backend.sdk.data.interops.SyncLogDataService; import org.dpppt.backend.sdk.interops.config.FlyWayConfig; import org.dpppt.backend.sdk.interops.config.GaenDataServiceConfig; +import org.dpppt.backend.sdk.interops.config.InteropsInsertManagerConfig; import org.dpppt.backend.sdk.interops.config.StandaloneDataConfig; import org.dpppt.backend.sdk.interops.config.SyncLogDataServiceConfig; +import org.dpppt.backend.sdk.interops.insertmanager.InteropsInsertManager; import org.dpppt.backend.sdk.interops.model.EfgsGatewayConfig; import org.dpppt.backend.sdk.interops.syncer.EfgsHubSyncer; import org.dpppt.backend.sdk.interops.syncer.efgs.EfgsClient; @@ -39,7 +41,8 @@ StandaloneDataConfig.class, FlyWayConfig.class, GaenDataServiceConfig.class, - SyncLogDataServiceConfig.class + SyncLogDataServiceConfig.class, + InteropsInsertManagerConfig.class }) public class SyncTest { @@ -47,6 +50,8 @@ public class SyncTest { @Autowired private SyncLogDataService syncLogDataService; + @Autowired private InteropsInsertManager interopsInsertManager; + private static final SecureRandom SECURE_RANDOM = new SecureRandom(); @Test @@ -68,14 +73,15 @@ public void testEfgsClientDownload() throws GeneralSecurityException { new EfgsClient(getEfgsGatewayConfig()), Duration.ofDays(14), gaenDataService, - syncLogDataService); + syncLogDataService, + interopsInsertManager); syncer.download(UTCInstant.today().getLocalDate()); } private EfgsGatewayConfig getEfgsGatewayConfig() { EfgsGatewayConfig efgsGatewayConfig = new EfgsGatewayConfig(); efgsGatewayConfig.setId("efgs-gateway"); - efgsGatewayConfig.setBaseUrl("https://api.ch-hub-r.bit.admin.ch"); + efgsGatewayConfig.setBaseUrl("https://api-ch-hub-r.bag.admin.ch"); efgsGatewayConfig.setAuthClientCert("base64:/*"); efgsGatewayConfig.setAuthClientCertPassword("*"); efgsGatewayConfig.setSignClientCert( @@ -102,6 +108,7 @@ private List createMockedKeys(int numOfKeysToCreate) { keyWithOrigin.setFake(0); keyWithOrigin.setOrigin("CH"); keyWithOrigin.setId(i); + keyWithOrigin.setReceivedAt(UTCInstant.now()); keys.add(keyWithOrigin); } return keys; diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/test/java/org/dpppt/backend/sdk/interops/config/InteropsInsertManagerConfig.java b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/test/java/org/dpppt/backend/sdk/interops/config/InteropsInsertManagerConfig.java new file mode 100644 index 00000000..0185bd93 --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-interops/src/test/java/org/dpppt/backend/sdk/interops/config/InteropsInsertManagerConfig.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2020 Ubique Innovation AG + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + * + * SPDX-License-Identifier: MPL-2.0 + */ + +package org.dpppt.backend.sdk.interops.config; + +import java.time.Duration; +import org.dpppt.backend.sdk.data.gaen.GaenDataService; +import org.dpppt.backend.sdk.interops.insertmanager.InteropsInsertManager; +import org.dpppt.backend.sdk.interops.insertmanager.insertionfilters.AssertKeyFormat; +import org.dpppt.backend.sdk.interops.insertmanager.insertionfilters.DsosFilter; +import org.dpppt.backend.sdk.interops.insertmanager.insertionfilters.EnforceRetentionPeriod; +import org.dpppt.backend.sdk.interops.insertmanager.insertionfilters.EnforceValidRollingPeriod; +import org.dpppt.backend.sdk.interops.insertmanager.insertionfilters.RemoveKeysFromFuture; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class InteropsInsertManagerConfig { + + @Value("${ws.retentiondays: 14}") + int retentionDays; + + @Value("${ws.app.gaen.key_size: 16}") + int gaenKeySizeBytes; + + @Bean + public InteropsInsertManager interopsInsertManager(GaenDataService gaenDataService) { + var manager = new InteropsInsertManager(gaenDataService); + manager.addFilter(new AssertKeyFormat(gaenKeySizeBytes)); + manager.addFilter(new RemoveKeysFromFuture()); + manager.addFilter(new EnforceRetentionPeriod(Duration.ofDays(retentionDays))); + manager.addFilter(new EnforceValidRollingPeriod()); + manager.addFilter(new DsosFilter()); + return manager; + } +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/InsertManager.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/InsertManager.java index 881b7a86..3470278c 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/InsertManager.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/InsertManager.java @@ -16,9 +16,9 @@ /** * The insertion manager is responsible for inserting keys uploaded by clients into the database. To * make sure we only have valid keys in the database, a list of {@link KeyInsertionModifier} is - * applied, and then a list of {@Link KeyInsertionFilter} is applied to the given list of keys. The + * applied, and then a list of {@link KeyInsertionFilter} is applied to the given list of keys. The * remaining keys are then inserted into the database. If any of the modifiers filters throws an - * {@Link InsertException} the process of insertions is aborted and the exception is propagated back + * {@link InsertException} the process of insertions is aborted and the exception is propagated back * to the caller, which is responsible for handling the exception. */ public class InsertManager { diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/insertmanager/MockDataSource.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/insertmanager/MockDataSource.java index 98bcb3c7..e2cd7ded 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/insertmanager/MockDataSource.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/insertmanager/MockDataSource.java @@ -44,7 +44,10 @@ public List getSortedExposedSince( @Override public void upsertExposeeFromInterops( - GaenKey key, UTCInstant now, String origin) { // TODO Auto-generated method stub + List keys, + UTCInstant now, + String origin, + String batchTag) { // TODO Auto-generated method stub } @Override