Merge pull request #1872 from cloudsufi/e2e-normalizePlugin-ITN

e2e-Normalize-ITN

core-plugins/src/e2e-test/features/normalize/FileToNormalize.feature

@@ -147,3 +147,52 @@ Feature: Normalize transform - Verify File source data transfer using Normalize
     Then Verify the pipeline status is "Succeeded"
     Then Close the pipeline logs
     Then Validate output file generated by file sink plugin "fileSinkTargetBucket" is equal to expected output file "normalizeCsvAllDataTypeOutputFile"
+
+  @NORMALIZE_TEST3 @FILE_SINK_TEST
+  Scenario: To verify data is getting transferred from File source to File sink successfully with Normalized fields using empty attribute
+    Given Open Datafusion Project to configure pipeline
+    When Select plugin: "File" from the plugins list as: "Source"
+    When Expand Plugin group in the LHS plugins list: "Transform"
+    When Select plugin: "Transpose" from the plugins list as: "Transform"
+    Then Connect plugins: "File" and "Transpose" to establish connection
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "File" from the plugins list as: "Sink"
+    Then Connect plugins: "Transpose" and "File2" to establish connection
+    Then Navigate to the properties page of plugin: "File"
+    Then Enter input plugin property: "referenceName" with value: "FileReferenceName"
+    Then Enter input plugin property: "path" with value: "normalizeTest3"
+    Then Select dropdown plugin property: "format" with option value: "csv"
+    Then Click plugin property: "skipHeader"
+    Then Click on the Get Schema button
+    Then Verify the Output Schema matches the Expected Schema: "normalizeCsvAllDataTypeFileSchema"
+    Then Validate "File" plugin properties
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "Transpose"
+    Then Enter Normalize plugin Fields to be Mapped "normalizeFileValidFieldsMapping"
+    Then Enter Normalize plugin Fields to be Normalized "normalizeFieldsToBeNormalizedFile"
+    Then Select Normalize plugin output schema action: "clear"
+    Then Enter Normalize plugin outputSchema "normalizeBQValidOutputSchema"
+    Then Validate "Transpose" plugin properties
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "File2"
+    Then Enter input plugin property: "referenceName" with value: "FileReferenceName"
+    Then Enter input plugin property: "path" with value: "fileSinkTargetBucket"
+    Then Replace input plugin property: "pathSuffix" with value: "yyyy-MM-dd-HH-mm-ss"
+    Then Select dropdown plugin property: "format" with option value: "csv"
+    Then Click plugin property: "writeHeader"
+    Then Validate "File2" plugin properties
+    Then Close the Plugin Properties page
+    Then Save the pipeline
+    Then Preview and run the pipeline
+    Then Wait till pipeline preview is in running state
+    Then Open and capture pipeline preview logs
+    Then Verify the preview run status of pipeline in the logs is "succeeded"
+    Then Close the pipeline logs
+    Then Close the preview
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Verify the pipeline status is "Succeeded"
+    Then Close the pipeline logs
+    Then Validate output file generated by file sink plugin "fileSinkTargetBucket" is equal to expected output file "normalizeCsvAllDataTypeOutputFile1"

core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java

@@ -244,7 +244,16 @@ public static void createBucketWithNormalizeTest1File() throws IOException, URIS
     BeforeActions.scenario.write("Normalize 1st bucket name - " + gcsSourceBucketName1);
   }
 
-  @After(order = 1, value = "@NORMALIZE_TEST1")
+  @Before(order = 1, value = "@NORMALIZE_TEST3")
+  public static void createBucketWithNormalizeTest3File() throws IOException, URISyntaxException {
+    gcsSourceBucketName1 = createGCSBucketWithFile(PluginPropertyUtils.pluginProp(
+      "normalizeCsvAllDataTypeFile3"));
+    PluginPropertyUtils.addPluginProp("normalizeTest3", "gs://" + gcsSourceBucketName1 + "/"  +
+      PluginPropertyUtils.pluginProp("normalizeCsvAllDataTypeFile3"));
+    BeforeActions.scenario.write("Normalize 1st bucket name - " + gcsSourceBucketName1);
+  }
+
+  @After(order = 1, value = "@NORMALIZE_TEST1 or @NORMALIZE_TEST3")
   public static void deleteSourceBucketWithNormalizeTest1File() {
     deleteGCSBucket(gcsSourceBucketName1);
     gcsSourceBucketName1 = StringUtils.EMPTY;

core-plugins/src/e2e-test/resources/pluginParameters.properties

@@ -131,7 +131,9 @@ normalizeCsvFileOutputSchema={ "type": "record", "name": "text", "fields": [ \
 normalizeBQValidDatatypeOutputSchema=[{"key":"ID","value":"int"},{"key":"AttributeType","value":"string"},\
   {"key":"AttributeValue","value":"string"},{"key":"Date","value":"string"}]
 normalizeCsvAllDataTypeFile=testdata/file/CSV_Normalize_TEST_1.csv
+normalizeCsvAllDataTypeFile3=testdata/file/CSV_Normalize_TEST_3.csv
 normalizeCsvAllDataTypeOutputFile=e2e-tests/expected_outputs/CSV_NormalizeOutput.csv
+normalizeCsvAllDataTypeOutputFile1=e2e-tests/file/expected_outputs/OUTPUT_FOR_NORMALIZE_TEST.csv
 normalizeCsvAllDataTypeFileSchema=[{"key":"id","value":"int"},{"key":"name","value":"string"},\
   {"key":"yearofbirth","value":"int"},{"key":"isdeleted","value":"boolean"},{"key":"email","value":"string"},\
   {"key":"createddate","value":"string"},{"key":"revenue","value":"string"},{"key":"points","value":"string"},\

core-plugins/src/e2e-test/resources/testdata/file/CSV_Normalize_TEST_3.csv

@@ -0,0 +1,5 @@
+id,name,yearofbirth,isdeleted,email,createddate,revenue,points,longdatatype,doubledatatype,date,null,BytesData
+1,albert einstein,1879,true,[email protected],,900750000.01,3.14235678,-9223372036854770000,22.8,1996-07-21,,10111011101110111011
+2,isaac newton,1643,false,[email protected],,900750000.01,3.14235678906787648,-9223372036854770000,123.08,1996-07-21,str,10111011101110111011
+3,marie curie,1867,true,[email protected],2021-09-20 11:27:50 UTC,900750000.01,3.14235678,-9223372036854770000,124.97,1996-07-21,,10111011101110111011
+4,galilée,1564,false,[email protected],2021-09-20 11:27:50 UTC,900750000.01,3.14235678,-2^63,234.89,1996-07-21,,10111011101110111011

core-plugins/src/e2e-test/resources/testdata/file/expected_outputs/OUTPUT_FOR_NORMALIZE_TEST.csv

@@ -0,0 +1,13 @@
+AttributeValue,AttributeType,ID,Date
+[email protected],email,1,null
+900750000.01,revenue,1,null
+3.14235678,points,1,null
+[email protected],email,2,null
+900750000.01,revenue,2,null
+3.14235678906787648,points,2,null
+[email protected],email,3,2021-09-20 11:27:50 UTC
+900750000.01,revenue,3,2021-09-20 11:27:50 UTC
+3.14235678,points,3,2021-09-20 11:27:50 UTC
+[email protected],email,4,2021-09-20 11:27:50 UTC
+900750000.01,revenue,4,2021-09-20 11:27:50 UTC
+3.14235678,points,4,2021-09-20 11:27:50 UTC