From 92b699d798c541a407828c39b7b63e412ed46ef6 Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Sat, 18 May 2024 14:02:57 +0200 Subject: [PATCH] disabled jupyter tests relying on DISPLAY(), update jupyter to latest java 8 version --- buildSrc/src/main/kotlin/Versions.kt | 8 ++-- gradle/bootstraps/compiler-plugin.jar | Bin 47032 -> 47032 bytes gradle/bootstraps/gradle-plugin.jar | Bin 9347 -> 9347 bytes .../kotlinx/spark/api/jupyter/JupyterTests.kt | 37 +++++++++--------- .../jetbrains/kotlinx/spark/api/Encoding.kt | 18 ++++++--- 5 files changed, 36 insertions(+), 27 deletions(-) diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt index ed3ce444..64b0d510 100644 --- a/buildSrc/src/main/kotlin/Versions.kt +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -2,7 +2,7 @@ object Versions : Dsl { const val project = "2.0.0-SNAPSHOT" const val kotlinSparkApiGradlePlugin = "2.0.0-SNAPSHOT" const val groupID = "org.jetbrains.kotlinx.spark" - const val kotlin = "2.0.0-RC1" + const val kotlin = "2.0.0-RC3" const val jvmTarget = "8" const val jupyterJvmTarget = "8" inline val spark get() = System.getProperty("spark") as String @@ -12,10 +12,10 @@ object Versions : Dsl { inline val scalaCompat get() = scala.substringBeforeLast('.') // TODO inline val sparkConnect get() = System.getProperty("sparkConnect", "false").toBoolean() - const val jupyter = "0.12.0-32-1" + const val jupyter = "0.12.0-163" // latest jupyter version with java 8 const val gradlePublishPlugin = "1.1.0" - const val kotest = "5.5.4" + const val kotest = "5.9.0" const val shadow = "8.1.1" const val buildconfig = "5.3.5" @@ -33,7 +33,7 @@ object Versions : Dsl { const val kotlinxHtml = "0.7.5" const val klaxon = "5.5" const val jacksonDatabind = "2.13.4.2" - const val kotlinxDateTime = "0.6.0-RC.2" + const val kotlinxDateTime = "0.6.0" inline val versionMap: Map get() = mapOf( diff --git a/gradle/bootstraps/compiler-plugin.jar b/gradle/bootstraps/compiler-plugin.jar index af6fb778737690d95bd62bfbe96d5a4fdf668ac4..fa1826de8afb94e8b131e9df9cf82993e8643427 100644 GIT binary patch delta 482 zcmdn-o@vK>Cf)#VW)=|!1`ZAeiM&k{dDWOfl$jTpYF+`NChIXOg6SYeZ7|)yXbh&e zFDOx^ z>Sfnig87l_9KrP5b?#vL`8sbfZMr@ROizN+@7KqH`2iayg6St4y1{hO#&$6MXCp*> z!X^k`YI7`DylQhTnEti70Zi9!se!29QU~F0tq0Tpw^o7anr*gVde62pFm1ZM833)y BucH6} delta 482 zcmdn-o@vK>Cf)#VW)=|!1`ZAe=NX+7dDWP;mQ+nN^I~=Y@|stGsL6VaieNg3Q5#G* zFdBpDZH$(JK=}v;AYcL!K%g-BqoJ}CGXug{kcz-giKH-Q28JtalOM3EZBAy&X9k;a zmDLAKtFQ%t=*{J9jBH>=HP1XIFhho4N(jubR5V}(Gjh~QxIv7`j}8670;WcP!odua zt~iM9sS|3rz#<~E6(E|-7c@hp_AWcY0`lVIj@9~Lde`dpU^-{bdS+k!X^k`YI7`DylQhTnEti70Zi9!se!29QU~F0tq0Tpw^o7OTeHm; OOz+uN2BuB7Hv<4{Z?`o7 diff --git a/gradle/bootstraps/gradle-plugin.jar b/gradle/bootstraps/gradle-plugin.jar index fe0cde250c326c54b81d7fb2fbc7f7483f384527..b470374e9ed9c4e7cfd9c91acf25b7a25752beae 100644 GIT binary patch delta 270 zcmZqnZ1&_0@MdNaVPN3kV35e$G?7=08AO?RfvM&&Fm*=}L``DU22qps7>&Vn3Zo^M zUchJzqBlQaT)+$#Xl4cL-h6?zgAFVb%_9t!*<8r~jSs9OOU_gfs3$^kUa&yBW0A|-$5dZ)H delta 270 zcmZqnZ1&_0@MdNaVPN3kV3^$9IgwY5S$j# createRepl( + httpUtil = createLibraryHttpUtil(), scriptClasspath = classpath, isEmbedded = true, ).apply { @@ -108,7 +110,7 @@ class JupyterTests : ShouldSpec({ sc as? JavaSparkContext shouldNotBe null } - should("render Datasets") { + xshould("render Datasets") { @Language("kts") val html = execForDisplayText( """ @@ -124,7 +126,7 @@ class JupyterTests : ShouldSpec({ html shouldContain "3" } - should("render JavaRDDs") { + xshould("render JavaRDDs") { @Language("kts") val html = execForDisplayText( """ @@ -141,7 +143,7 @@ class JupyterTests : ShouldSpec({ html shouldContain "4, 5, 6" } - should("render JavaRDDs with Arrays") { + xshould("render JavaRDDs with Arrays") { @Language("kts") val html = execForDisplayText( """ @@ -158,7 +160,7 @@ class JupyterTests : ShouldSpec({ html shouldContain "4, 5, 6" } - should("render JavaRDDs with custom class") { + xshould("render JavaRDDs with custom class") { @Language("kts") val klass = exec( @@ -192,7 +194,7 @@ class JupyterTests : ShouldSpec({ +-------------+---------------+--------------------+""".trimIndent() } - should("render JavaPairRDDs") { + xshould("render JavaPairRDDs") { @Language("kts") val html = execForDisplayText( """ @@ -214,7 +216,7 @@ class JupyterTests : ShouldSpec({ +---+---+""".trimIndent() } - should("render JavaDoubleRDD") { + xshould("render JavaDoubleRDD") { @Language("kts") val html = execForDisplayText( """ @@ -230,7 +232,7 @@ class JupyterTests : ShouldSpec({ html shouldContain "4.0" } - should("render Scala RDD") { + xshould("render Scala RDD") { @Language("kts") val html = execForDisplayText( """ @@ -247,7 +249,7 @@ class JupyterTests : ShouldSpec({ html shouldContain "4, 5, 6" } - should("truncate dataset cells using properties") { + xshould("truncate dataset cells using properties") { @Language("kts") val oldTruncation = exec("""sparkProperties.displayTruncate""") as Int @@ -268,7 +270,7 @@ class JupyterTests : ShouldSpec({ html shouldNotContain "aaaaaaaaaa" } - should("limit dataset rows using properties") { + xshould("limit dataset rows using properties") { @Language("kts") val oldLimit = exec("""sparkProperties.displayLimit""") as Int @@ -292,7 +294,7 @@ class JupyterTests : ShouldSpec({ html shouldNotContain "e|" } - should("truncate rdd cells using properties") { + xshould("truncate rdd cells using properties") { @Language("kts") val oldTruncation = exec("""sparkProperties.displayTruncate""") as Int @@ -312,7 +314,7 @@ class JupyterTests : ShouldSpec({ html shouldNotContain "aaaaaaaaaa" } - should("limit rdd rows using properties") { + xshould("limit rdd rows using properties") { @Language("kts") val oldLimit = exec("""sparkProperties.displayLimit""") as Int @@ -344,6 +346,7 @@ class JupyterTests : ShouldSpec({ class JupyterStreamingTests : ShouldSpec({ val replProvider = ReplProvider { classpath -> createRepl( + httpUtil = createLibraryHttpUtil(), scriptClasspath = classpath, isEmbedded = true, ).apply { @@ -373,7 +376,7 @@ class JupyterStreamingTests : ShouldSpec({ fun createRepl(): ReplForJupyter = replProvider(scriptClasspath) suspend fun withRepl(action: suspend ReplForJupyter.() -> Unit): Unit = createRepl().action() - context("Jupyter") { + xcontext("Jupyter") { withRepl { // For when onInterrupt is implemented in the Jupyter kernel @@ -449,9 +452,7 @@ class JupyterStreamingTests : ShouldSpec({ private fun ReplForJupyter.execEx(code: Code): EvalResultEx = evalEx(EvalRequestData(code)) -private fun ReplForJupyter.exec(code: Code): Any? = execEx(code).renderedValue - -private fun ReplForJupyter.execRaw(code: Code): Any? = execEx(code).rawValue +private fun ReplForJupyter.exec(code: Code): Any? = (execEx(code) as? EvalResultEx.Success)?.renderedValue @JvmName("execTyped") private inline fun ReplForJupyter.exec(code: Code): T { diff --git a/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt index 874b5e66..0306b8f6 100644 --- a/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt +++ b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt @@ -304,11 +304,19 @@ object KotlinTypeInference : Serializable { a + b.mapValues { a.getOrDefault(valueToKey(it.value), it.value) } private fun registerUdts() { - UDTRegistration.register(kotlinx.datetime.LocalDate::class.java.name, LocalDateUdt::class.java.name) - UDTRegistration.register(kotlinx.datetime.Instant::class.java.name, InstantUdt::class.java.name) - UDTRegistration.register(kotlinx.datetime.LocalDateTime::class.java.name, LocalDateTimeUdt::class.java.name) - UDTRegistration.register(kotlinx.datetime.DatePeriod::class.java.name, DatePeriodUdt::class.java.name) - UDTRegistration.register(kotlinx.datetime.DateTimePeriod::class.java.name, DateTimePeriodUdt::class.java.name) + val udts = listOf( + kotlinx.datetime.LocalDate::class to LocalDateUdt::class, + kotlinx.datetime.Instant::class to InstantUdt::class, + kotlinx.datetime.LocalDateTime::class to LocalDateTimeUdt::class, + kotlinx.datetime.DatePeriod::class to DatePeriodUdt::class, + kotlinx.datetime.DateTimePeriod::class to DateTimePeriodUdt::class, + ) + + for ((kClass, udtClass) in udts) { + if (!UDTRegistration.exists(kClass.java.name)) { + UDTRegistration.register(kClass.java.name, udtClass.java.name) + } + } // TODO // UDTRegistration.register(kotlin.time.Duration::class.java.name, DurationUdt::class.java.name) }