MOSIP-30305 compliance_testrun_summary added (#93)

* removed extra col Signed-off-by: Mayura Deshmukh <[email protected]> * MOSIP-29738 updated TC dashboard Signed-off-by: Mayura Deshmukh <[email protected]> * MOSIP-30408 updated ctk quality reports as new cat Signed-off-by: Mayura Deshmukh <[email protected]> * MOSIP-30305 compliance_testrun_summary added Signed-off-by: Mayura Deshmukh <[email protected]> --------- Signed-off-by: Mayura Deshmukh <[email protected]>
mosip · Nov 28, 2023 · ce52b16 · ce52b16
1 parent 3b3c4f8
commit ce52b16
Show file tree

Hide file tree

Showing 3 changed files with 73 additions and 2 deletions.
diff --git a/kafka-connect/debez-sample-conn.api b/kafka-connect/debez-sample-conn.api
@@ -33,7 +33,7 @@ curl \
           "database.dbname": '\"$DB_NAME\"',
           "database.server.name": '\"$DB_PREFIX_INDEX\"',
           "table.include.list": '\"$DB_TABLES\"',
-          "column.exclude.list":"toolkit.sbi_projects.sbi_hash,toolkit.sbi_projects.device_image1,toolkit.sbi_projects.device_image2,toolkit.sbi_projects.device_image3,toolkit.sbi_projects.device_image4,toolkit.abis_projects.username,toolkit.abis_projects.password,toolkit.abis_projects.abis_hash,toolkit.sdk_projects.sdk_hash,toolkit.test_run_details.method_url,toolkit.test_run_details.method_request,toolkit.test_run_details.method_response",
+          "column.exclude.list":"toolkit.sbi_projects.sbi_hash,toolkit.sbi_projects.device_image1,toolkit.sbi_projects.device_image2,toolkit.sbi_projects.device_image3,toolkit.sbi_projects.device_image4,toolkit.abis_projects.username,toolkit.abis_projects.password,toolkit.abis_projects.abis_hash,toolkit.sdk_projects.sdk_hash,toolkit.test_run_details.method_url,toolkit.test_run_details.method_request,toolkit.test_run_details.method_response,toolkit.compliance_testrun_summary.report_data_json",
           "heartbeat.interval.ms": '\"$CONNECTOR_HEARTBEAT_MS\"'
       }
 }';

diff --git a/kafka-connect/ref_connector_api_calls/71.toolkit_compliance_testrun_summary.api b/kafka-connect/ref_connector_api_calls/71.toolkit_compliance_testrun_summary.api
@@ -0,0 +1,71 @@
+# these following will be taken from env
+#DB_USER=
+#DB_PORT=
+#DB_HOSTNAME=
+#DB_PASS=
+#DB_PREFIX_INDEX=
+#ES_URL=
+
+CONN_NAME="toolkit_compliance_testrun_summary_$DB_PREFIX_INDEX"; # change this.. give unique name for each db/table
+
+ES_CONN_URL='es-connect:8083'; # needn't change .. this is the ES-connector service name
+
+DB_NAME='mosip_toolkit'; # change this
+DB_TABLES='toolkit.compliance_testrun_summary'
+ES_INDICES=$(echo $DB_TABLES | sed -E "s/([^,]+)/$DB_PREFIX_INDEX.\1/g")
+
+curl \
+  -X POST \
+  http://$ES_CONN_URL/connectors \
+  -H 'Content-Type: application/json' \
+  -d \
+  '{
+      "name": '\"$CONN_NAME\"',
+      "config": {
+          "connector.class": "io.confluent.connect.elasticsearch.ElasticsearchSinkConnector",
+          "name": '\"$CONN_NAME\"',
+          "connection.url": '\"$ES_URL\"',
+          "tasks.max": "1",
+          "topics": '\"$ES_INDICES\"',
+          "key.ignore": "false",
+          "schema.ignore": "true",
+          "key.converter": "org.apache.kafka.connect.storage.StringConverter",
+          "value.converter": "org.apache.kafka.connect.json.JsonConverter",
+          "key.converter.schemas.enable": "false",
+          "value.converter.schemas.enable": "true",
+
+          "behavior.on.null.values": "DELETE",
+
+          "transforms": "debezExtract,tsconvert01,tsconvert02,tsconvert03,tsconvert04,tsconvert05,tsSelect",
+
+          "transforms.debezExtract.type": "io.debezium.transforms.ExtractNewRecordState",
+          "transforms.debezExtract.delete.handling.mode": "drop",
+          "transforms.debezExtract.drop.tombstones": "false",
+          "transforms.debezExtract.add.fields": "source.ts_ms:ts_ms,table,lsn",
+          "transforms.debezExtract.add.fields.prefix": "source_",
+
+          "transforms.tsconvert01.type": "io.mosip.kafka.connect.transforms.TimestampConverterAdv$Value",
+          "transforms.tsconvert01.field": "source_ts_ms",
+
+          "transforms.tsconvert02.type": "io.mosip.kafka.connect.transforms.TimestampConverterAdv$Value",
+          "transforms.tsconvert02.field": "review_dttimes",
+          "transforms.tsconvert02.input.type": "micro_sec",
+
+          "transforms.tsconvert03.type": "io.mosip.kafka.connect.transforms.TimestampConverterAdv$Value",
+          "transforms.tsconvert03.field": "approve_reject_dttimes",
+          "transforms.tsconvert03.input.type": "micro_sec",
+
+          "transforms.tsconvert04.type": "io.mosip.kafka.connect.transforms.TimestampConverterAdv$Value",
+          "transforms.tsconvert04.field": "cr_dtimes",
+          "transforms.tsconvert04.input.type": "micro_sec",
+
+          "transforms.tsconvert05.type": "io.mosip.kafka.connect.transforms.TimestampConverterAdv$Value",
+          "transforms.tsconvert05.field": "upd_dtimes",
+          "transforms.tsconvert05.input.type": "micro_sec",
+
+          "transforms.tsSelect.type": "io.mosip.kafka.connect.transforms.TimestampSelector$Value",
+          "transforms.tsSelect.ts.order": "upd_dtimes,cr_dtimes",
+          "transforms.tsSelect.output.field": "@timestamp_gen"
+
+      }
+}';
diff --git a/scripts/values-init.yaml b/scripts/values-init.yaml
@@ -26,7 +26,7 @@ debezium_connectors:
   - name: "mosip_idrepo"
     tables: "idrepo.anonymous_profile,idrepo.channel_info"
   - name: "mosip_toolkit"
-    tables: "toolkit.sbi_projects,toolkit.sdk_projects,toolkit.abis_projects,toolkit.collections,toolkit.collection_testcase_mapping,toolkit.testcase,toolkit.test_run,toolkit.test_run_details,toolkit.biometric_scores"    
+    tables: "toolkit.sbi_projects,toolkit.sdk_projects,toolkit.abis_projects,toolkit.collections,toolkit.collection_testcase_mapping,toolkit.testcase,toolkit.test_run,toolkit.test_run_details,toolkit.biometric_scores,toolkit.compliance_testrun_summary"    
 
 es_kafka_connectors:
   enabled: true