fix: sdk deps & nor-informed generator (#37)

* fix: sdk deps & nor-informed generator

* fix: generator constructor

app/build.gradle

@@ -38,6 +38,7 @@ dependencies {
     implementation group: 'com.googlecode.libphonenumber', name: 'libphonenumber', version: '8.12.25-4p'
     implementation group: 'com.github.javafaker', name: 'javafaker', version: '1.0.2'
     implementation group: 'com.googlecode.java-ipv6', name: 'java-ipv6', version: '0.17'
+    implementation group: 'com.google.code.gson', name: 'gson', version: '2.10.1'
 
     testImplementation group: 'org.apache.spark', name: 'spark-sql_2.12', version: '3.3.2'
     testImplementation group: 'commons-validator', name: 'commons-validator', version: '1.6'

app/src/main/java/com/telefonica/baikal/utils/avro/NotInformedUtilsBridge.java

@@ -0,0 +1,15 @@
+package com.telefonica.baikal.utils.avro;
+
+public class NotInformedUtilsBridge implements NotInformedUtils {
+
+    private static NotInformedUtilsBridge instance = null;
+
+    public static synchronized NotInformedUtilsBridge getInstance() {
+       if (instance == null) {
+           instance = new NotInformedUtilsBridge();
+           return instance;
+       }
+       return instance;
+    }
+
+}

app/src/main/java/io/confluent/avro/random/generator/Generator.java

@@ -18,7 +18,7 @@
 
 import com.mifmif.common.regex.Generex;
 import com.telefonica.baikal.avro.types.CustomLogicalTypes;
-import com.telefonica.baikal.utils.avro.BaikalAvroUtils;
+import com.telefonica.baikal.utils.avro.NotInformedUtilsBridge;
 import org.apache.avro.LogicalType;
 import org.apache.avro.LogicalTypes;
 import org.apache.avro.Schema;
@@ -306,7 +306,7 @@ public Builder schemaFile(File schemaFile) throws IOException {
     public Builder schemaFile(File schemaFile, Boolean useNotInformedSchema) throws IOException {
       if (useNotInformedSchema) {
         String rawSchema = Files.readString(schemaFile.toPath());
-        String notInformedSchema = BaikalAvroUtils.createNotInformedSchema(rawSchema).toString();
+        String notInformedSchema = NotInformedUtilsBridge.getInstance().createNotInformedSchema(rawSchema).toString();
         System.out.println(notInformedSchema);
         topLevelSchema = parser.parse(notInformedSchema);
       } else {
@@ -326,7 +326,7 @@ public Builder schemaString(String schemaString) {
 
     public Builder schemaString(String schemaString, Boolean useNotInformedSchema) {
       if (useNotInformedSchema) {
-        String notInformedSchema = BaikalAvroUtils.createNotInformedSchema(schemaString).toString();
+        String notInformedSchema = NotInformedUtilsBridge.getInstance().createNotInformedSchema(schemaString).toString();
         System.out.println(notInformedSchema);
         topLevelSchema = parser.parse(notInformedSchema);
       } else {

app/src/main/java/io/confluent/avro/random/generator/Main.java

@@ -324,19 +324,25 @@ private static Generator getGenerator(String schema, String schemaFile) throws I
         return getGenerator(schema, schemaFile, Optional.empty(), Optional.empty());
     }
 
-    private static Generator getGenerator(String schema, String schemaFile, Optional<Double> malformedNotInformedRate, Optional<Double> malformedColumnRate) throws IOException {
+    private static Generator getGenerator(String schema, String schemaFile, Optional<Double> notInformedColumnRate, Optional<Double> malformedColumnRate) throws IOException {
         if (schema != null) {
             return new Generator.Builder().schemaString(schema)
                     .malformedColumnRate(malformedColumnRate)
+                    .notInformedColumnRate(notInformedColumnRate)
                     .build();
         } else if (!schemaFile.equals("-")) {
             return new Generator.Builder()
-                    .schemaFile(new File(schemaFile), malformedNotInformedRate.isPresent())
+                    .schemaFile(new File(schemaFile), notInformedColumnRate.isPresent())
                     .malformedColumnRate(malformedColumnRate)
+                    .notInformedColumnRate(notInformedColumnRate)
                     .build();
         } else {
             System.err.println("Reading schema from stdin...");
-            return new Generator.Builder().schemaStream(System.in).malformedColumnRate(malformedColumnRate).build();
+            return new Generator.Builder()
+                    .schemaStream(System.in)
+                    .malformedColumnRate(malformedColumnRate)
+                    .notInformedColumnRate(notInformedColumnRate)
+                    .build();
         }
     }
 

app/src/test/java/io/confluent/avro/random/generator/CustomExtensionsGeneratorTest.java

@@ -104,9 +104,9 @@ record = (GenericRecord) generator.generate();
       }
     }
 
-    assertEquals("Wrong string distribution", 0.45, ((double) stringResults.size()) / 100, 0.1);
-    assertEquals("Wrong int distribution", 0.45, ((double) intResults.size()) / 100, 0.1);
-    assertEquals("Wrong not informed distribution", 0.1, ((double) notInformedResults.size()) / 100, 0.1);
+    assertEquals("Wrong string distribution", 0.45, ((double) stringResults.size()) / 100, 0.2);
+    assertEquals("Wrong int distribution", 0.45, ((double) intResults.size()) / 100, 0.2);
+    assertEquals("Wrong not informed distribution", 0.1, ((double) notInformedResults.size()) / 100, 0.2);
     assertNotEquals("Empty not informed values", notInformedResults.size(), 0);
   }
 

lanuza/base.inc

@@ -2,7 +2,7 @@
 
 #@IgnoreInspection BashAddShebang
 #
-# LANUZA v4.3.3
+# LANUZA v4.14.0
 #
 # This file provides:
 # - a default control flow
@@ -75,23 +75,37 @@ function fail {
 
 # gets the lanuza root dir
 # https://www.ostricher.com/2014/10/the-right-way-to-get-the-directory-of-a-bash-script/
-function get_root_dir () {
+function get_root_dir {
   local SOURCE DIR
 
-  SOURCE="${BASH_SOURCE[0]}"
-  # While $SOURCE is a symlink, resolve it
-  while [ -h "$SOURCE" ]; do
-    DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-    SOURCE="$( readlink "$SOURCE" )"
-    # If $SOURCE was a relative symlink (so no "/" as prefix, need to resolve it relative to the symlink base directory
-    [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
-  done
+  # absolute pathname of this script
+  SOURCE="$( _realpath "${BASH_SOURCE[0]}" )"
 
-  # once this file has
+  # print absolute pathname of parent directory
   DIR="$( cd -P "$( dirname "$SOURCE" )/../" && pwd )"
   echo "$DIR"
 }
 
+# gets the canonicalised absolute pathname of the argument, resolving symlinks
+# realpath(1) is part of GNU Coreutils, not always available as preinstalled command in other Unix variants
+function _realpath {
+  local NAME="$1"
+  local DIR
+
+  # While $NAME is a symlink, resolve it
+  while [[ -h "$NAME" ]]; do
+    DIR="$( cd -P "$( dirname "$NAME" )" && pwd )"
+    NAME="$( readlink "$NAME" )"
+    # If $NAME was a relative symlink (so no "/" as prefix), need to resolve it relative to the symlink base directory
+    [[ $NAME != /* ]] && NAME="$DIR/$NAME"
+  done
+
+  # Resolve relative paths in $NAME and print final absolute pathname
+  [[ -d "$NAME" ]] && NAME="$( cd -P "$NAME" && pwd )"
+  DIR="$( cd -P "$( dirname "$NAME" )" && pwd )"
+  echo "${DIR%/}/$( basename "${NAME%/}" )"
+}
+
 function parseArgs {
   local REQUIRED_ARG_NAMES=()
 
@@ -172,7 +186,7 @@ function parseArgs {
 
     # -- run
     for i in "$@"; do
-      parseArg $i
+      parseArg "$i"
       validateArg
       setVar "${ARG_NAME_UPPER}" "$ARG_VALUE"
     done
@@ -223,68 +237,72 @@ function isFunction {
 function gen_lanuza_id() {
   # Using /dev/urandom hangs the process on GitHub Actions, and we do not need such great entropy
   # based on https://gist.github.com/markusfisch/6110640
-	local N B C='89ab'
-
-	for (( N=0; N < 6; ++N )); do
-		B=$(( $RANDOM%256 ))
-
-		case $N in
-			1)
-				printf '4%x' $(( B%16 ))
-				;;
-			3 | 5)
-				printf '%c%x' ${C:$RANDOM%${#C}:1} $(( B%16 ))
-				;;
-			*)
-				printf '%02x' $B
-				;;
-		esac
-	done
-
-	echo
+  local N B C='89ab'
+
+  for (( N=0; N < 6; ++N )); do
+    B=$(( $RANDOM%256 ))
+
+    case $N in
+      1)
+        printf '4%x' $(( B%16 ))
+        ;;
+      3 | 5)
+        printf '%c%x' ${C:$RANDOM%${#C}:1} $(( B%16 ))
+        ;;
+      *)
+        printf '%02x' $B
+        ;;
+    esac
+  done
+
+  echo
 }
 
 function main {
   function cleanup__() {
+    local exit_code=$?
+
     mapfile -t cleanup_functions < <( declare -F | grep -E 'cleanup$' | tr -s ' ' | cut -d ' ' -f3 )
 
     for cleanup in "${cleanup_functions[@]}"; do
-      $cleanup || true
+      $cleanup $exit_code|| true
     done
   }
 
   set -Eeo pipefail
   set +x
 
   setVar "LANUZA_ROOT" "$(get_root_dir)"
+
+  # caller 1 # contains the caller script
+  local source=$(_realpath $(caller 1 | tr -s ' ' | cut -d ' ' -f3 ))
+  local rootless_source="${source#"${LANUZA_ROOT}/"}"
+  setVar "LANUZA_SOURCE" "$rootless_source"
+  setVar "LANUZA_SOURCE_FILE" "$(basename $source .sh)"
+
+  # a random number tied to this execution
+  setVar "LANUZA_BUILD_ID" "${LANUZA_BUILD_ID:-$(gen_lanuza_id)}"
+  setVar "LANUZA_UUID" "$(gen_lanuza_id)"
+
+  # Output directory to save lanuza generated files
+  setVar "LANUZA_OUTPUT_DIR" "${LANUZA_OUTPUT_DIR:-"output"}"
+
   # change directory to base directory of the project
   BASE_DIR=$(dirname $BASH_SOURCE)
   cd $BASE_DIR/..;
 
-  local current_dir=$(pwd)
+  local current_dir=$(_realpath $PWD)
   local project_name
   if [[ ${LANUZA_ROOT} == ${current_dir} ]]; then
-    # use the root dir as wht project name
-    project_name=$(basename "$(pwd)")
+    # use the root dir as project name
+    project_name=$(basename "${LANUZA_ROOT}")
   else
     # The relative path from root to project without /
     project_name=$(echo ${current_dir#"${LANUZA_ROOT}/"} | tr '/' _ )
   fi
 
   setVar "LANUZA_PROJECT" "${project_name}"
 
-  # caller 1 # contains the caller script
-  local source=$(caller 1 | tr -s ' ' | cut -d ' ' -f3 )
-  setVar "LANUZA_SOURCE" "$source"
-  setVar "LANUZA_SOURCE_FILE" "$(basename $source .sh)"
-
-  # a random number tied to this execution
-  setVar "LANUZA_BUILD_ID" "${LANUZA_BUILD_ID:-$(gen_lanuza_id)}"
-  setVar "LANUZA_UUID" "$(gen_lanuza_id)"
-
-  # Output directory to save lanuza generated files
-  setVar "LANUZA_OUTPUT_DIR" "${LANUZA_OUTPUT_DIR:-"output"}"
-
   local f
   # load other utilities
   for f in $(find ${LANUZA_ROOT}/lanuza/utils -name '*.inc'); do

lanuza/pipelines/build.sh

@@ -11,8 +11,8 @@ function init() {
 }
 
 function run() {
-  docker-compose build app
-  docker-compose run app ./lanuza/scripts/build.sh
+  docker compose build app
+  docker compose run app ./lanuza/scripts/build.sh
 }
 
 source $(dirname $0)/../base.inc

lanuza/pipelines/run.sh

@@ -7,8 +7,8 @@ function init() {
 }
 
 function run() {
-  docker-compose build app 1>&2
-  docker-compose run app "$@"
+  docker compose build app 1>&2
+  docker compose run app "$@"
 }
 
 source $(dirname $0)/../base.inc

lanuza/utils/artifact.inc

@@ -81,6 +81,20 @@ function artifact_exists_helm() {
   _artifact_check_with_retries exists "no such manifest"
 }
 
+# Checks if an artifact exists in an OCI registry
+# uses the docker cli to check the manifest
+function artifact_exists_oci() {
+  local image="${1}"
+
+  function exists() {
+    docker manifest inspect "${image}"
+  }
+
+  # Docker error on missing artifact is exit code == 1 and stderr:
+  # no such manifest: ${image}
+  _artifact_check_with_retries exists "no such manifest"
+}
+
 # Retries an exists command a with backoff.
 #
 # The retry count is given by LANUZA_ARTIFACT_RETRIES (default 5), the

lanuza/utils/docker-compose.inc

@@ -146,7 +146,8 @@ function _docker-compose_build() {
     debug "Using the following docker-compose file to specify labels"
     debug "$yaml"
     # Include new yaml in the docker-compose files
-    temp_compose=$(mktemp "${TMPDIR:-/tmp/}"docker-compose.labels.$LANUZA_BUILD_ID.XXXXXXXXX)
+    temp_compose=$(: ${TMPDIR:=/tmp}; mktemp "${TMPDIR%/}/docker-compose.labels.$LANUZA_BUILD_ID.XXXXXXXXX")
+
     # To cleanup at exit
     temp_files+=( $temp_compose )
     echo "$yaml" > $temp_compose
@@ -218,7 +219,8 @@ function _docker-compose_build() {
     debug "$yaml"
 
     # Include new yaml in the docker-compose files
-    temp_compose=$(mktemp "${TMPDIR:-/tmp/}"docker-compose.cache.$LANUZA_BUILD_ID.XXXXXXXXX)
+    temp_compose=$(: ${TMPDIR:=/tmp}; mktemp "${TMPDIR%/}/docker-compose.cache.$LANUZA_BUILD_ID.XXXXXXXXX")
+
     # To cleanup at exit
     temp_files+=( $temp_compose )
     echo "$yaml" > $temp_compose