CDAP-21027 : Upgrade to Hadoop 3.3.6

cdapio · Jul 26, 2024 · 5b44445 · 5b44445
1 parent fa4eb00
commit 5b44445
Show file tree

Hide file tree

Showing 16 changed files with 73 additions and 134 deletions.
diff --git a/amazon-s3-plugins b/amazon-s3-plugins
diff --git a/cassandra-plugins/pom.xml b/cassandra-plugins/pom.xml
@@ -66,16 +66,6 @@
       <groupId>org.apache.cassandra</groupId>
       <artifactId>cassandra-all</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.codehaus.jackson</groupId>
-      <artifactId>jackson-core-asl</artifactId>
-      <version>1.9.13</version>
-    </dependency>
-    <dependency>
-      <groupId>org.codehaus.jackson</groupId>
-      <artifactId>jackson-mapper-asl</artifactId>
-      <version>1.9.13</version>
-    </dependency>
     <dependency>
       <groupId>org.jboss.netty</groupId>
       <artifactId>netty</artifactId>

diff --git a/core-plugins/pom.xml b/core-plugins/pom.xml
@@ -209,10 +209,6 @@
       <artifactId>sshd-core</artifactId>
       <version>1.2.0</version>
     </dependency>
-    <dependency>
-      <groupId>org.apache.hive</groupId>
-      <artifactId>hive-exec</artifactId>
-    </dependency>
     <dependency>
       <groupId>io.cdap.plugin</groupId>
       <artifactId>format-avro</artifactId>

diff --git a/database-plugins/pom.xml b/database-plugins/pom.xml
@@ -75,7 +75,7 @@
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
       <scope>test</scope>
-      <version>2.13.4.2</version>
+      <version>${jackson.databind.version}</version>
     </dependency>
     <dependency>
       <groupId>junit</groupId>

diff --git a/format-avro/pom.xml b/format-avro/pom.xml
@@ -68,14 +68,9 @@
       direct dependency for some reason
        -->
     <dependency>
-      <groupId>org.codehaus.jackson</groupId>
-      <artifactId>jackson-core-asl</artifactId>
-      <version>1.9.13</version>
-    </dependency>
-    <dependency>
-      <groupId>org.codehaus.jackson</groupId>
-      <artifactId>jackson-mapper-asl</artifactId>
-      <version>1.9.13</version>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+      <version>${jackson.databind.version}</version>
     </dependency>
     <dependency>
       <groupId>junit</groupId>

diff --git a/format-delimited/src/main/java/io/cdap/plugin/format/delimited/common/DateTimePattern.java b/format-delimited/src/main/java/io/cdap/plugin/format/delimited/common/DateTimePattern.java
@@ -20,7 +20,6 @@
 import io.cdap.plugin.format.delimited.dto.DateTimeStandard;
 import io.cdap.plugin.format.delimited.dto.SupportedDateTimeStandards;
 import org.apache.commons.lang3.StringUtils;
-import org.codehaus.jackson.map.ObjectMapper;
 
 import java.io.BufferedReader;
 import java.io.InputStream;

diff --git a/format-parquet/pom.xml b/format-parquet/pom.xml
@@ -81,24 +81,15 @@
       <artifactId>parquet-avro</artifactId>
       <version>1.12.0</version>
     </dependency>
-    <!--
-      jackson is required by avro's schema parser but doesn't get bundled unless it's a
-      direct dependency for some reason
-       -->
-    <dependency>
-      <groupId>org.codehaus.jackson</groupId>
-      <artifactId>jackson-core-asl</artifactId>
-      <version>1.9.13</version>
-    </dependency>
-    <dependency>
-      <groupId>org.codehaus.jackson</groupId>
-      <artifactId>jackson-mapper-asl</artifactId>
-      <version>1.9.13</version>
-    </dependency>
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+      <version>${jackson.databind.version}</version>
+    </dependency>
   </dependencies>
 
 

diff --git a/format-parquet/src/main/java/org/apache/parquet/avro/AvroSchemaConverter.java b/format-parquet/src/main/java/org/apache/parquet/avro/AvroSchemaConverter.java
@@ -16,7 +16,6 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
 package org.apache.parquet.avro;
 
 import org.apache.avro.LogicalType;
@@ -32,7 +31,6 @@
 import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
 import org.apache.parquet.schema.Type;
 import org.apache.parquet.schema.Types;
-import org.apache.parquet.schema.LogicalTypeAnnotation.UUIDLogicalTypeAnnotation;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -44,12 +42,8 @@
 import static java.util.Optional.empty;
 import static java.util.Optional.of;
 import static org.apache.avro.JsonProperties.NULL_VALUE;
-import static org.apache.parquet.avro.AvroReadSupport.READ_INT96_AS_FIXED;
-import static org.apache.parquet.avro.AvroReadSupport.READ_INT96_AS_FIXED_DEFAULT;
 import static org.apache.parquet.avro.AvroWriteSupport.WRITE_OLD_LIST_STRUCTURE;
 import static org.apache.parquet.avro.AvroWriteSupport.WRITE_OLD_LIST_STRUCTURE_DEFAULT;
-import static org.apache.parquet.avro.AvroWriteSupport.WRITE_PARQUET_UUID;
-import static org.apache.parquet.avro.AvroWriteSupport.WRITE_PARQUET_UUID_DEFAULT;
 import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.MICROS;
 import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.MILLIS;
 import static org.apache.parquet.schema.LogicalTypeAnnotation.dateType;
@@ -58,7 +52,6 @@
 import static org.apache.parquet.schema.LogicalTypeAnnotation.stringType;
 import static org.apache.parquet.schema.LogicalTypeAnnotation.timeType;
 import static org.apache.parquet.schema.LogicalTypeAnnotation.timestampType;
-import static org.apache.parquet.schema.LogicalTypeAnnotation.uuidType;
 import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.*;
 import static org.apache.parquet.schema.Type.Repetition.REPEATED;
 
@@ -78,11 +71,10 @@ public class AvroSchemaConverter {
 
   private final boolean assumeRepeatedIsListElement;
   private final boolean writeOldListStructure;
-  private final boolean writeParquetUUID;
-  private final boolean readInt96AsFixed;
 
   public AvroSchemaConverter() {
-    this(ADD_LIST_ELEMENT_RECORDS_DEFAULT);
+    this.assumeRepeatedIsListElement = ADD_LIST_ELEMENT_RECORDS_DEFAULT;
+    this.writeOldListStructure = WRITE_OLD_LIST_STRUCTURE_DEFAULT;
   }
 
   /**
@@ -94,17 +86,13 @@ public AvroSchemaConverter() {
   AvroSchemaConverter(boolean assumeRepeatedIsListElement) {
     this.assumeRepeatedIsListElement = assumeRepeatedIsListElement;
     this.writeOldListStructure = WRITE_OLD_LIST_STRUCTURE_DEFAULT;
-    this.writeParquetUUID = WRITE_PARQUET_UUID_DEFAULT;
-    this.readInt96AsFixed = READ_INT96_AS_FIXED_DEFAULT;
   }
 
   public AvroSchemaConverter(Configuration conf) {
     this.assumeRepeatedIsListElement = conf.getBoolean(
       ADD_LIST_ELEMENT_RECORDS, ADD_LIST_ELEMENT_RECORDS_DEFAULT);
     this.writeOldListStructure = conf.getBoolean(
       WRITE_OLD_LIST_STRUCTURE, WRITE_OLD_LIST_STRUCTURE_DEFAULT);
-    this.writeParquetUUID = conf.getBoolean(WRITE_PARQUET_UUID, WRITE_PARQUET_UUID_DEFAULT);
-    this.readInt96AsFixed = conf.getBoolean(READ_INT96_AS_FIXED, READ_INT96_AS_FIXED_DEFAULT);
   }
 
   /**
@@ -159,7 +147,6 @@ private Type convertField(String fieldName, Schema schema) {
   private Type convertField(String fieldName, Schema schema, Type.Repetition repetition) {
     Types.PrimitiveBuilder<PrimitiveType> builder;
     Schema.Type type = schema.getType();
-    LogicalType logicalType = schema.getLogicalType();
     if (type.equals(Schema.Type.BOOLEAN)) {
       builder = Types.primitive(BOOLEAN, repetition);
     } else if (type.equals(Schema.Type.INT)) {
@@ -173,12 +160,7 @@ private Type convertField(String fieldName, Schema schema, Type.Repetition repet
     } else if (type.equals(Schema.Type.BYTES)) {
       builder = Types.primitive(BINARY, repetition);
     } else if (type.equals(Schema.Type.STRING)) {
-      if (logicalType != null && logicalType.getName().equals(LogicalTypes.uuid().getName()) && writeParquetUUID) {
-        builder = Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition)
-          .length(LogicalTypeAnnotation.UUIDLogicalTypeAnnotation.BYTES);
-      } else {
-        builder = Types.primitive(BINARY, repetition).as(stringType());
-      }
+      builder = Types.primitive(BINARY, repetition).as(stringType());
     } else if (type.equals(Schema.Type.RECORD)) {
       return new GroupType(repetition, fieldName, convertFields(schema.getFields()));
     } else if (type.equals(Schema.Type.ENUM)) {
@@ -206,6 +188,7 @@ private Type convertField(String fieldName, Schema schema, Type.Repetition repet
 
     // schema translation can only be done for known logical types because this
     // creates an equivalence
+    LogicalType logicalType = schema.getLogicalType();
     if (logicalType != null) {
       if (logicalType instanceof LogicalTypes.Decimal) {
         LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
@@ -326,12 +309,8 @@ public Schema convertDOUBLE(PrimitiveTypeName primitiveTypeName) {
           }
           @Override
           public Schema convertFIXED_LEN_BYTE_ARRAY(PrimitiveTypeName primitiveTypeName) {
-            if (annotation instanceof LogicalTypeAnnotation.UUIDLogicalTypeAnnotation) {
-              return Schema.create(Schema.Type.STRING);
-            } else {
-              int size = parquetType.asPrimitiveType().getTypeLength();
-              return Schema.createFixed(parquetType.getName(), null, null, size);
-            }
+            int size = parquetType.asPrimitiveType().getTypeLength();
+            return Schema.createFixed(parquetType.getName(), null, null, size);
           }
           @Override
           public Schema convertBINARY(PrimitiveTypeName primitiveTypeName) {
@@ -443,8 +422,6 @@ private LogicalTypeAnnotation convertLogicalType(LogicalType logicalType) {
       return timestampType(true, MILLIS);
     } else if (logicalType instanceof LogicalTypes.TimestampMicros) {
       return timestampType(true, MICROS);
-    } else if (logicalType.getName().equals(LogicalTypes.uuid().getName()) && writeParquetUUID) {
-      return uuidType();
     }
     return null;
   }
@@ -487,11 +464,6 @@ public Optional<LogicalType> visit(LogicalTypeAnnotation.TimestampLogicalTypeAnn
         }
         return empty();
       }
-
-      @Override
-      public Optional<LogicalType> visit(UUIDLogicalTypeAnnotation uuidLogicalType) {
-        return of(LogicalTypes.uuid());
-      }
     }).orElse(null);
   }
 

diff --git a/google-cloud b/google-cloud
diff --git a/hbase-plugins/pom.xml b/hbase-plugins/pom.xml
@@ -68,7 +68,7 @@
     <dependency>
       <groupId>org.apache.hbase</groupId>
       <artifactId>hbase-server</artifactId>
-      <version>0.98.6.1-hadoop2</version>
+      <version>2.5.8-hadoop3</version>
       <exclusions>
         <exclusion>
           <groupId>org.apache.hbase</groupId>
@@ -127,7 +127,7 @@
     <dependency>
       <groupId>org.apache.hbase</groupId>
       <artifactId>hbase-testing-util</artifactId>
-      <version>0.98.6.1-hadoop2</version>
+      <version>2.5.8-hadoop3</version>
       <scope>test</scope>
       <exclusions>
         <exclusion>

diff --git a/http-plugins/pom.xml b/http-plugins/pom.xml
@@ -68,7 +68,7 @@
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
       <scope>test</scope>
-      <version>2.13.4.2</version>
+      <version>${jackson.databind.version}</version>
     </dependency>
   </dependencies>
 

diff --git a/hydrator-common/src/main/java/io/cdap/plugin/common/spark/VectorUtils.java b/hydrator-common/src/main/java/io/cdap/plugin/common/spark/VectorUtils.java
@@ -17,7 +17,7 @@
 
 import io.cdap.cdap.api.data.format.StructuredRecord;
 import io.cdap.cdap.api.data.schema.Schema;
-import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang3.ArrayUtils;
 import org.apache.spark.mllib.linalg.SparseVector;
 
 import java.util.List;

diff --git a/mongodb-plugins/pom.xml b/mongodb-plugins/pom.xml
@@ -96,6 +96,25 @@
     <dependency>
       <groupId>org.apache.hive</groupId>
       <artifactId>hive-exec</artifactId>
+      <version>3.1.3</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.codehaus.jackson</groupId>
+          <artifactId>jackson-mapper-asl</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.codehaus.jackson</groupId>
+          <artifactId>jackson-core-asl</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.calcite.avatica</groupId>
+          <artifactId>avatica</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
   </dependencies>