diff --git a/kafka-connect/ref_connector_api_calls/71.toolkit_biometric_scores_new.api b/kafka-connect/ref_connector_api_calls/71.toolkit_biometric_scores_new.api new file mode 100644 index 0000000..234c362 --- /dev/null +++ b/kafka-connect/ref_connector_api_calls/71.toolkit_biometric_scores_new.api @@ -0,0 +1,56 @@ +# these following will be taken from env +#DB_USER= +#DB_PORT= +#DB_HOSTNAME= +#DB_PASS= +#DB_PREFIX_INDEX= +#ES_URL= + +CONN_NAME="toolkit_biometric_scores_new_$DB_PREFIX_INDEX"; # change this.. give unique name for each db/table + +ES_CONN_URL='es-connect:8083'; # needn't change .. this is the ES-connector service name + +DB_NAME='mosip_toolkit'; # change this +DB_TABLES='toolkit.biometric_scores_new' +ES_INDICES=$(echo $DB_TABLES | sed -E "s/([^,]+)/$DB_PREFIX_INDEX.\1/g") + +curl \ + -X POST \ + http://$ES_CONN_URL/connectors \ + -H 'Content-Type: application/json' \ + -d \ + '{ + "name": '\"$CONN_NAME\"', + "config": { + "connector.class": "io.confluent.connect.elasticsearch.ElasticsearchSinkConnector", + "name": '\"$CONN_NAME\"', + "connection.url": '\"$ES_URL\"', + "tasks.max": "1", + "topics": '\"$ES_INDICES\"', + "key.ignore": "false", + "schema.ignore": "true", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "key.converter.schemas.enable": "false", + "value.converter.schemas.enable": "true", + + "behavior.on.null.values": "DELETE", + + "transforms": "debezExtract,tsconvert01,tsSelect", + + "transforms.debezExtract.type": "io.debezium.transforms.ExtractNewRecordState", + "transforms.debezExtract.delete.handling.mode": "drop", + "transforms.debezExtract.drop.tombstones": "false", + "transforms.debezExtract.add.fields": "source.ts_ms:ts_ms,table,lsn", + "transforms.debezExtract.add.fields.prefix": "source_", + + "transforms.tsconvert01.type": "io.mosip.kafka.connect.transforms.TimestampConverterAdv$Value", + "transforms.tsconvert01.field": "cr_dtimes", + "transforms.tsconvert01.input.type": "micro_sec", + + "transforms.tsSelect.type": "io.mosip.kafka.connect.transforms.TimestampSelector$Value", + "transforms.tsSelect.ts.order": "cr_dtimes", + "transforms.tsSelect.output.field": "@timestamp_gen" + + } +}'; diff --git a/scripts/values-init.yaml b/scripts/values-init.yaml index 1f551cb..a305f0a 100644 --- a/scripts/values-init.yaml +++ b/scripts/values-init.yaml @@ -26,7 +26,7 @@ debezium_connectors: - name: "mosip_idrepo" tables: "idrepo.anonymous_profile,idrepo.channel_info" - name: "mosip_toolkit" - tables: "toolkit.sbi_projects,toolkit.sdk_projects,toolkit.abis_projects,toolkit.collections,toolkit.collection_testcase_mapping,toolkit.testcase,toolkit.test_run,toolkit.test_run_details,toolkit.biometric_scores" + tables: "toolkit.sbi_projects,toolkit.sdk_projects,toolkit.abis_projects,toolkit.collections,toolkit.collection_testcase_mapping,toolkit.testcase,toolkit.test_run,toolkit.test_run_details,toolkit.biometric_scores,toolkit.biometric_scores_new" es_kafka_connectors: enabled: true