diff --git a/kafka-connect/debez-sample-conn.api b/kafka-connect/debez-sample-conn.api index 7a5c6e7..59bcebe 100644 --- a/kafka-connect/debez-sample-conn.api +++ b/kafka-connect/debez-sample-conn.api @@ -33,7 +33,7 @@ curl \ "database.dbname": '\"$DB_NAME\"', "database.server.name": '\"$DB_PREFIX_INDEX\"', "table.include.list": '\"$DB_TABLES\"', - "column.exclude.list":"toolkit.sbi_projects.sbi_hash,toolkit.sbi_projects.device_image1,toolkit.sbi_projects.device_image2,toolkit.sbi_projects.device_image3,toolkit.sbi_projects.device_image4,toolkit.abis_projects.username,toolkit.abis_projects.password,toolkit.abis_projects.abis_hash,toolkit.sdk_projects.sdk_hash,toolkit.test_run_details.method_url,toolkit.test_run_details.method_request,toolkit.test_run_details.method_response", + "column.exclude.list":"toolkit.sbi_projects.sbi_hash,toolkit.sbi_projects.device_image1,toolkit.sbi_projects.device_image2,toolkit.sbi_projects.device_image3,toolkit.sbi_projects.device_image4,toolkit.abis_projects.username,toolkit.abis_projects.password,toolkit.abis_projects.abis_hash,toolkit.sdk_projects.sdk_hash,toolkit.test_run_details.method_url,toolkit.test_run_details.method_request,toolkit.test_run_details.method_response,toolkit.compliance_testrun_summary.report_data_json", "heartbeat.interval.ms": '\"$CONNECTOR_HEARTBEAT_MS\"' } }'; diff --git a/kafka-connect/ref_connector_api_calls/71.toolkit_compliance_testrun_summary.api b/kafka-connect/ref_connector_api_calls/71.toolkit_compliance_testrun_summary.api new file mode 100644 index 0000000..8a4ca9d --- /dev/null +++ b/kafka-connect/ref_connector_api_calls/71.toolkit_compliance_testrun_summary.api @@ -0,0 +1,71 @@ +# these following will be taken from env +#DB_USER= +#DB_PORT= +#DB_HOSTNAME= +#DB_PASS= +#DB_PREFIX_INDEX= +#ES_URL= + +CONN_NAME="toolkit_compliance_testrun_summary_$DB_PREFIX_INDEX"; # change this.. give unique name for each db/table + +ES_CONN_URL='es-connect:8083'; # needn't change .. this is the ES-connector service name + +DB_NAME='mosip_toolkit'; # change this +DB_TABLES='toolkit.compliance_testrun_summary' +ES_INDICES=$(echo $DB_TABLES | sed -E "s/([^,]+)/$DB_PREFIX_INDEX.\1/g") + +curl \ + -X POST \ + http://$ES_CONN_URL/connectors \ + -H 'Content-Type: application/json' \ + -d \ + '{ + "name": '\"$CONN_NAME\"', + "config": { + "connector.class": "io.confluent.connect.elasticsearch.ElasticsearchSinkConnector", + "name": '\"$CONN_NAME\"', + "connection.url": '\"$ES_URL\"', + "tasks.max": "1", + "topics": '\"$ES_INDICES\"', + "key.ignore": "false", + "schema.ignore": "true", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "key.converter.schemas.enable": "false", + "value.converter.schemas.enable": "true", + + "behavior.on.null.values": "DELETE", + + "transforms": "debezExtract,tsconvert01,tsconvert02,tsconvert03,tsconvert04,tsconvert05,tsSelect", + + "transforms.debezExtract.type": "io.debezium.transforms.ExtractNewRecordState", + "transforms.debezExtract.delete.handling.mode": "drop", + "transforms.debezExtract.drop.tombstones": "false", + "transforms.debezExtract.add.fields": "source.ts_ms:ts_ms,table,lsn", + "transforms.debezExtract.add.fields.prefix": "source_", + + "transforms.tsconvert01.type": "io.mosip.kafka.connect.transforms.TimestampConverterAdv$Value", + "transforms.tsconvert01.field": "source_ts_ms", + + "transforms.tsconvert02.type": "io.mosip.kafka.connect.transforms.TimestampConverterAdv$Value", + "transforms.tsconvert02.field": "review_dttimes", + "transforms.tsconvert02.input.type": "micro_sec", + + "transforms.tsconvert03.type": "io.mosip.kafka.connect.transforms.TimestampConverterAdv$Value", + "transforms.tsconvert03.field": "approve_reject_dttimes", + "transforms.tsconvert03.input.type": "micro_sec", + + "transforms.tsconvert04.type": "io.mosip.kafka.connect.transforms.TimestampConverterAdv$Value", + "transforms.tsconvert04.field": "cr_dtimes", + "transforms.tsconvert04.input.type": "micro_sec", + + "transforms.tsconvert05.type": "io.mosip.kafka.connect.transforms.TimestampConverterAdv$Value", + "transforms.tsconvert05.field": "upd_dtimes", + "transforms.tsconvert05.input.type": "micro_sec", + + "transforms.tsSelect.type": "io.mosip.kafka.connect.transforms.TimestampSelector$Value", + "transforms.tsSelect.ts.order": "upd_dtimes,cr_dtimes", + "transforms.tsSelect.output.field": "@timestamp_gen" + + } +}'; diff --git a/scripts/values-init.yaml b/scripts/values-init.yaml index 1f551cb..21fb062 100644 --- a/scripts/values-init.yaml +++ b/scripts/values-init.yaml @@ -26,7 +26,7 @@ debezium_connectors: - name: "mosip_idrepo" tables: "idrepo.anonymous_profile,idrepo.channel_info" - name: "mosip_toolkit" - tables: "toolkit.sbi_projects,toolkit.sdk_projects,toolkit.abis_projects,toolkit.collections,toolkit.collection_testcase_mapping,toolkit.testcase,toolkit.test_run,toolkit.test_run_details,toolkit.biometric_scores" + tables: "toolkit.sbi_projects,toolkit.sdk_projects,toolkit.abis_projects,toolkit.collections,toolkit.collection_testcase_mapping,toolkit.testcase,toolkit.test_run,toolkit.test_run_details,toolkit.biometric_scores,toolkit.compliance_testrun_summary" es_kafka_connectors: enabled: true