From 11599591c6a9fe82d336695075b46224c3e824c9 Mon Sep 17 00:00:00 2001 From: Kristin Cowalcijk Date: Mon, 19 Jun 2023 15:46:32 +0800 Subject: [PATCH 1/7] Fixed maven prepare-release by including all submodules when executing prepare release --- pom.xml | 12 ++++++++++++ sql/pom.xml | 12 ++++++++++++ sql/spark-3.0/pom.xml | 2 +- sql/spark-3.4/pom.xml | 2 +- 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index d1ddef4f65..e76905b37c 100644 --- a/pom.xml +++ b/pom.xml @@ -360,6 +360,18 @@ maven-shade-plugin 3.4.1 + + org.apache.maven.plugins + maven-release-plugin + 3.0.1 + + all + true + + !enable-all-submodules + + net.alchim31.maven scala-maven-plugin diff --git a/sql/pom.xml b/sql/pom.xml index beabf12112..9f52f40e87 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -41,6 +41,18 @@ spark-${spark.compat.version} + + + + enable-all-submodules + + spark-3.4 + + + + diff --git a/sql/spark-3.0/pom.xml b/sql/spark-3.0/pom.xml index 39d51a2a64..c5ab649087 100644 --- a/sql/spark-3.0/pom.xml +++ b/sql/spark-3.0/pom.xml @@ -25,7 +25,7 @@ 1.4.1-SNAPSHOT ../pom.xml - sedona-sql-${spark.compat.version}_${scala.compat.version} + sedona-sql-3.0_${scala.compat.version} ${project.groupId}:${project.artifactId} A cluster computing system for processing large-scale spatial data: SQL API for Spark 3.0 - 3.3. diff --git a/sql/spark-3.4/pom.xml b/sql/spark-3.4/pom.xml index 531d449828..ca18129e69 100644 --- a/sql/spark-3.4/pom.xml +++ b/sql/spark-3.4/pom.xml @@ -25,7 +25,7 @@ 1.4.1-SNAPSHOT ../pom.xml - sedona-sql-${spark.compat.version}_${scala.compat.version} + sedona-sql-3.4_${scala.compat.version} ${project.groupId}:${project.artifactId} A cluster computing system for processing large-scale spatial data: SQL API for Spark 3.4. From 9a2b13be99a6b67402d9da4b64b11c4450b1dda7 Mon Sep 17 00:00:00 2001 From: Kristin Cowalcijk Date: Mon, 19 Jun 2023 23:29:30 +0800 Subject: [PATCH 2/7] Don't release scala doc for sedona-core and sedona-viz --- core/pom.xml | 8 ++++++++ viz/pom.xml | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/core/pom.xml b/core/pom.xml index 3e16e36235..be76e82961 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -183,6 +183,14 @@ net.alchim31.maven scala-maven-plugin + + + attach-javadocs + + true + + + org.scalatest diff --git a/viz/pom.xml b/viz/pom.xml index 69eb062e7b..0f9824499b 100644 --- a/viz/pom.xml +++ b/viz/pom.xml @@ -155,6 +155,14 @@ net.alchim31.maven scala-maven-plugin + + + attach-javadocs + + true + + + org.scalatest From b74e86607cecaf5f81c1d4677624871c47f3a999 Mon Sep 17 00:00:00 2001 From: Kristin Cowalcijk Date: Tue, 20 Jun 2023 00:15:19 +0800 Subject: [PATCH 3/7] Don't run scala maven plugin for sedona-flink --- flink/pom.xml | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/flink/pom.xml b/flink/pom.xml index 1c52fa141a..1451c0899f 100644 --- a/flink/pom.xml +++ b/flink/pom.xml @@ -121,12 +121,6 @@ scala-library ${scala.version} - - org.scalatest - scalatest_${scala.compat.version} - ${scalatest.version} - test - org.apache.logging.log4j log4j-1.2-api @@ -145,21 +139,5 @@ compile - - - - net.alchim31.maven - scala-maven-plugin - - - org.scalatest - scalatest-maven-plugin - - - org.scalastyle - scalastyle-maven-plugin - - - From 15a2645d9ff0ffc62248b209eb4209d443417b86 Mon Sep 17 00:00:00 2001 From: Kristin Cowalcijk Date: Tue, 20 Jun 2023 01:08:28 +0800 Subject: [PATCH 4/7] Don't recalculate signatures after deploy --- pom.xml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/pom.xml b/pom.xml index e76905b37c..cbf5433bb3 100644 --- a/pom.xml +++ b/pom.xml @@ -488,15 +488,6 @@ org.apache.maven.plugins maven-gpg-plugin 3.0.1 - - - sign-artifacts - deploy - - sign - - - net.nicoulaj.maven.plugins From 361be499826ad4e923aec225711b4db5c631a81e Mon Sep 17 00:00:00 2001 From: Kristin Cowalcijk Date: Tue, 20 Jun 2023 10:50:04 +0800 Subject: [PATCH 5/7] [DOCS] Upgrade Python, R, Zeppelin versions to 1.4.1 --- R/DESCRIPTION | 2 +- R/R/dependencies.R | 2 +- python/sedona/version.py | 2 +- zeppelin/package.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/R/DESCRIPTION b/R/DESCRIPTION index 8fd8b9a049..2b91d66046 100644 --- a/R/DESCRIPTION +++ b/R/DESCRIPTION @@ -1,7 +1,7 @@ Type: Package Package: apache.sedona Title: R Interface for Apache Sedona -Version: 1.4.0.9000 +Version: 1.4.1 Authors@R: c(person(family = "Apache Sedona", role = c("aut", "cre"), diff --git a/R/R/dependencies.R b/R/R/dependencies.R index bead3f09b2..fc38e84869 100644 --- a/R/R/dependencies.R +++ b/R/R/dependencies.R @@ -39,7 +39,7 @@ spark_dependencies <- function(spark_version, scala_version, ...) { paste0( "org.apache.sedona:sedona-", c("spark-shaded", "viz"), - sprintf("-%s_%s:1.4.0", spark_version, scala_version) + sprintf("-%s_%s:1.4.1", spark_version, scala_version) ), packages ) diff --git a/python/sedona/version.py b/python/sedona/version.py index da2562beab..4da8e7157a 100644 --- a/python/sedona/version.py +++ b/python/sedona/version.py @@ -15,4 +15,4 @@ # specific language governing permissions and limitations # under the License. -version = "1.4.0" +version = "1.4.1" diff --git a/zeppelin/package.json b/zeppelin/package.json index 4e4c09a4bb..8f2cd0fbe2 100644 --- a/zeppelin/package.json +++ b/zeppelin/package.json @@ -2,7 +2,7 @@ "name": "apache-sedona", "description": "Zeppelin visualization support for Sedona", "author": "Apache Sedona, original authors are listed on https://github.com/myuwono/zeppelin-leaflet", - "version": "1.4.0", + "version": "1.4.1", "main": "index.js", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", From 0c7279f1948eead2cd47f4b30a792d339f849189 Mon Sep 17 00:00:00 2001 From: Kristin Cowalcijk Date: Tue, 20 Jun 2023 13:39:16 +0800 Subject: [PATCH 6/7] [DOCS] Update docs for release --- docs/community/publish.md | 161 ++++++++++++++---------------------- docs/community/snapshot.md | 30 +++---- docs/setup/release-notes.md | 111 +++++++++++++++++++++++++ mkdocs.yml | 10 +-- 4 files changed, 192 insertions(+), 120 deletions(-) diff --git a/docs/community/publish.md b/docs/community/publish.md index dfdb5404a6..bf524643a4 100644 --- a/docs/community/publish.md +++ b/docs/community/publish.md @@ -9,7 +9,7 @@ This page is for Sedona PMC to publish Sedona releases. 1. In your local Sedona Git repo under master branch, run ```bash -echo "#!/bin/bash" > create-release.sh +echo '#!/bin/bash' > create-release.sh chmod 777 create-release.sh ``` 2. Use your favourite GUI text editor to open `create-release.sh`. @@ -61,8 +61,6 @@ Make sure the Sedona version in the following files are {{ sedona_create_release ```bash #!/bin/bash -source ~/.bashrc - git checkout master git pull @@ -71,20 +69,31 @@ rm -f pom.xml.* echo "*****Step 1. Stage the Release Candidate to GitHub." -mvn -q -B clean release:prepare -Dtag={{ sedona_create_release.current_git_tag }} -DreleaseVersion={{ sedona_create_release.current_version }} -DdevelopmentVersion={{ sedona_create_release.current_snapshot }} -DautoVersionSubmodules=true -Dresume=false -Darguments="-DskipTests" - -rm -f release.* -rm -f pom.xml.* +mvn -q -B clean release:prepare -Dtag={{ sedona_create_release.current_git_tag }} -DreleaseVersion={{ sedona_create_release.current_version }} -DdevelopmentVersion={{ sedona_create_release.current_snapshot }} -Dresume=false -Penable-all-submodules -Darguments="-DskipTests" +mvn -q -B release:clean -Penable-all-submodules echo "Now the releases are staged. A tag and two commits have been created on Sedona GitHub repo" echo "*****Step 2: Upload the Release Candidate to https://repository.apache.org." # For Spark 3.0 and Scala 2.12 -mvn -q org.apache.maven.plugins:maven-release-plugin:2.3.2:perform -DconnectionUrl=scm:git:https://github.com/apache/sedona.git -Dtag={{ sedona_create_release.current_git_tag }} -DautoVersionSubmodules=true -Dresume=false -Darguments="-DskipTests -Dscala=2.12" +mvn -q org.apache.maven.plugins:maven-release-plugin:2.3.2:perform -DconnectionUrl=scm:git:https://github.com/apache/sedona.git -Dtag={{ sedona_create_release.current_git_tag }} -Dresume=false -Darguments="-DskipTests -Dspark=3.0 -Dscala=2.12" -Dspark=3.0 -Dscala=2.12 # For Spark 3.0 and Scala 2.13 -mvn -q org.apache.maven.plugins:maven-release-plugin:2.3.2:perform -DconnectionUrl=scm:git:https://github.com/apache/sedona.git -Dtag={{ sedona_create_release.current_git_tag }} -DautoVersionSubmodules=true -Dresume=false -Darguments="-DskipTests -Dscala=2.13" +## Note that we use maven-release-plugin 2.3.2 instead of more recent version (e.g., 3.0.1) to get rid of a bug of maven-release-plugin, +## which prevent us from cloning git repo with user specified -Dtag=. +## Please refer to https://issues.apache.org/jira/browse/MRELEASE-933 and https://issues.apache.org/jira/browse/SCM-729 for details. +## +## Please also note that system properties `-Dspark` and `-Dscala` has to be specified both for release:perform and the actual build parameters +## in `-Darguments`, because the build profiles activated for release:perform task will also affect the actual build task. It is safer to specify +## these system properties for both tasks. +mvn -q org.apache.maven.plugins:maven-release-plugin:2.3.2:perform -DconnectionUrl=scm:git:https://github.com/apache/sedona.git -Dtag={{ sedona_create_release.current_git_tag }} -Dresume=false -Darguments="-DskipTests -Dspark=3.0 -Dscala=2.13" -Dspark=3.0 -Dscala=2.13 + +# For Spark 3.4 and Scala 2.12 +mvn -q org.apache.maven.plugins:maven-release-plugin:2.3.2:perform -DconnectionUrl=scm:git:https://github.com/apache/sedona.git -Dtag={{ sedona_create_release.current_git_tag }} -Dresume=false -Darguments="-DskipTests -Dspark=3.4 -Dscala=2.12" -Dspark=3.4 -Dscala=2.12 + +# For Spark 3.4 and Scala 2.13 +mvn -q org.apache.maven.plugins:maven-release-plugin:2.3.2:perform -DconnectionUrl=scm:git:https://github.com/apache/sedona.git -Dtag={{ sedona_create_release.current_git_tag }} -Dresume=false -Darguments="-DskipTests -Dspark=3.4 -Dscala=2.13" -Dspark=3.4 -Dscala=2.13 echo "*****Step 3: Upload Release Candidate on ASF SVN: https://dist.apache.org/repos/dist/dev/sedona" @@ -108,19 +117,33 @@ echo "Compiling the source code..." mkdir apache-sedona-{{ sedona_create_release.current_version }}-bin -cd apache-sedona-{{ sedona_create_release.current_version }}-src && mvn -q clean install -DskipTests -Dscala=2.12 && cd .. +cd apache-sedona-{{ sedona_create_release.current_version }}-src && mvn -q clean install -DskipTests -Dspark=3.0 -Dscala=2.12 && cd .. cp apache-sedona-{{ sedona_create_release.current_version }}-src/common/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ cp apache-sedona-{{ sedona_create_release.current_version }}-src/core/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ -cp apache-sedona-{{ sedona_create_release.current_version }}-src/sql/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ +cp apache-sedona-{{ sedona_create_release.current_version }}-src/sql/spark-3.0/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ cp apache-sedona-{{ sedona_create_release.current_version }}-src/viz/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ cp apache-sedona-{{ sedona_create_release.current_version }}-src/python-adapter/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ cp apache-sedona-{{ sedona_create_release.current_version }}-src/spark-shaded/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ cp apache-sedona-{{ sedona_create_release.current_version }}-src/flink/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ cp apache-sedona-{{ sedona_create_release.current_version }}-src/flink-shaded/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ -cd apache-sedona-{{ sedona_create_release.current_version }}-src && mvn -q clean install -DskipTests -Dscala=2.13 && cd .. +cd apache-sedona-{{ sedona_create_release.current_version }}-src && mvn -q clean install -DskipTests -Dspark=3.0 -Dscala=2.13 && cd .. cp apache-sedona-{{ sedona_create_release.current_version }}-src/core/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ -cp apache-sedona-{{ sedona_create_release.current_version }}-src/sql/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ +cp apache-sedona-{{ sedona_create_release.current_version }}-src/sql/spark-3.0/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ +cp apache-sedona-{{ sedona_create_release.current_version }}-src/viz/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ +cp apache-sedona-{{ sedona_create_release.current_version }}-src/python-adapter/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ +cp apache-sedona-{{ sedona_create_release.current_version }}-src/spark-shaded/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ + +cd apache-sedona-{{ sedona_create_release.current_version }}-src && mvn -q clean install -DskipTests -Dspark=3.4 -Dscala=2.12 && cd .. +cp apache-sedona-{{ sedona_create_release.current_version }}-src/core/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ +cp apache-sedona-{{ sedona_create_release.current_version }}-src/sql/spark-3.4/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ +cp apache-sedona-{{ sedona_create_release.current_version }}-src/viz/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ +cp apache-sedona-{{ sedona_create_release.current_version }}-src/python-adapter/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ +cp apache-sedona-{{ sedona_create_release.current_version }}-src/spark-shaded/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ + +cd apache-sedona-{{ sedona_create_release.current_version }}-src && mvn -q clean install -DskipTests -Dspark=3.4 -Dscala=2.13 && cd .. +cp apache-sedona-{{ sedona_create_release.current_version }}-src/core/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ +cp apache-sedona-{{ sedona_create_release.current_version }}-src/sql/spark-3.4/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ cp apache-sedona-{{ sedona_create_release.current_version }}-src/viz/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ cp apache-sedona-{{ sedona_create_release.current_version }}-src/python-adapter/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ cp apache-sedona-{{ sedona_create_release.current_version }}-src/spark-shaded/target/sedona-*{{ sedona_create_release.current_version}}.jar apache-sedona-{{ sedona_create_release.current_version }}-bin/ @@ -312,103 +335,39 @@ rm apache-sedona-{{ sedona_create_release.current_version }}-src.tar.gz.sha512 rm apache-sedona-{{ sedona_create_release.current_version }}-bin.tar.gz rm apache-sedona-{{ sedona_create_release.current_version }}-bin.tar.gz.asc rm apache-sedona-{{ sedona_create_release.current_version }}-bin.tar.gz.sha512 +``` -echo "Re-staging releases to https://repository.apache.org" - -# For Spark 3.0 and Scala 2.12 - -git pull - -git checkout -b {{ sedona_create_release.current_git_tag }} {{ sedona_create_release.current_git_tag }} - -mvn versions:set -DnewVersion={{ sedona_create_release.current_version }}-SNAPSHOT - -git add -A - -git commit -m "tmp SNAPSHOT pom" - -mvn clean release:prepare -DautoVersionSubmodules=true -DdryRun=true -Dresume=false -Darguments="-DskipTests" -Dtag={{ sedona_create_release.current_git_tag }} -DreleaseVersion={{ sedona_create_release.current_version }} -DdevelopmentVersion={{ sedona_create_release.next_version }} - -mvn versions:set -DnewVersion={{ sedona_create_release.current_version }} - -git add -A - -git commit -m "revert back to the correct pom" - -mvn clean release:perform -DautoVersionSubmodules=true -Dresume=false -Darguments="-DskipTests" - -rm -f release.* -rm -f pom.xml.* - -# For Spark 3.0 and Scala 2.13 - -mvn versions:set -DnewVersion={{ sedona_create_release.current_version }}-SNAPSHOT - -git add -A - -git commit -m "tmp SNAPSHOT pom" +### Manually close and release the package -mvn clean release:prepare -DautoVersionSubmodules=true -DdryRun=true -Dresume=false -Darguments="-DskipTests -Dscala=2.13" -Dtag={{ sedona_create_release.current_git_tag }} -DreleaseVersion={{ sedona_create_release.current_version }} -DdevelopmentVersion={{ sedona_create_release.next_version }} +1. Click `Close` on the Sedona staging repo on https://repository.apache.org under `staging repository` +2. Once the staging repo is closed, click `Release` on this repo. -mvn versions:set -DnewVersion={{ sedona_create_release.current_version }} +**NOTICE**: The staging repo will be automatically dropped after 3 days without closing. If you find the staging repo being dropped, you can re-stage the release using the following script. -git add -A +```bash +#!/bin/bash -git commit -m "revert back to the correct pom" +echo "Re-staging releases to https://repository.apache.org" -mvn clean release:perform -DautoVersionSubmodules=true -Dresume=false -Darguments="-DskipTests -Dscala=2.13" +git checkout master +git pull rm -f release.* rm -f pom.xml.* -git add -A - -git commit -m "cleanup the branch" - -git checkout master - -git branch -d {{ sedona_create_release.current_git_tag }} -``` - -### Fix signature issues +# For Spark 3.0 and Scala 2.12 +mvn -q org.apache.maven.plugins:maven-release-plugin:2.3.2:perform -DconnectionUrl=scm:git:https://github.com/apache/sedona.git -Dtag={{ sedona_create_release.current_git_tag }} -Dresume=false -Darguments="-DskipTests -Dspark=3.0 -Dscala=2.12" -Dspark=3.0 -Dscala=2.12 -Please find the Sedona staging id on https://repository.apache.org under `staging repository`. +# For Spark 3.0 and Scala 2.13 +mvn -q org.apache.maven.plugins:maven-release-plugin:2.3.2:perform -DconnectionUrl=scm:git:https://github.com/apache/sedona.git -Dtag={{ sedona_create_release.current_git_tag }} -Dresume=false -Darguments="-DskipTests -Dspark=3.0 -Dscala=2.13" -Dspark=3.0 -Dscala=2.13 -Then run the following script. Replace `admin`, `admind123` with your Apache ID username and Apache ID password. Replace `stagingid` with the correct id. +# For Spark 3.4 and Scala 2.12 +mvn -q org.apache.maven.plugins:maven-release-plugin:2.3.2:perform -DconnectionUrl=scm:git:https://github.com/apache/sedona.git -Dtag={{ sedona_create_release.current_git_tag }} -Dresume=false -Darguments="-DskipTests -Dspark=3.4 -Dscala=2.12" -Dspark=3.4 -Dscala=2.12 -```bash -#!/bin/bash -username=admin -password=admin123 -stagingid=1027 - -artifacts=(parent core-3.0_2.12 core-3.0_2.13 sql-3.0_2.12 sql-3.0_2.13 viz-3.0_2.12 viz-3.0_2.13 python-adapter-3.0_2.12 python-adapter-3.0_2.13 common flink_2.12 spark-shaded-3.0_2.12 spark-shaded-3.0_2.13 flink-shaded_2.12) -filenames=(.pom .jar -javadoc.jar) - -echo "Re-uploading signatures to fix *failureMessage Invalid Signature*" -for artifact in "${artifacts[@]}"; do - for filename in "${filenames[@]}"; do - if [ $artifact -eq 'parent' && $filename -ne '.pom' ] - then - continue - fi - wget https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-$artifact/{{ sedona_create_release.current_version }}/sedona-${artifact}-{{ sedona_create_release.current_version }}${filename} - gpg -ab sedona-${artifact}-{{ sedona_create_release.current_version }}${filename} - curl -v -u $username:$password --upload-file sedona-${artifact}-{{ sedona_create_release.current_version }}${filename}.asc https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-${artifact}/{{ sedona_create_release.current_version }}/sedona-${artifact}-{{ sedona_create_release.current_version }}${filename}.asc - done -done - -rm *.pom -rm *.jar -rm *.asc +# For Spark 3.4 and Scala 2.13 +mvn -q org.apache.maven.plugins:maven-release-plugin:2.3.2:perform -DconnectionUrl=scm:git:https://github.com/apache/sedona.git -Dtag={{ sedona_create_release.current_git_tag }} -Dresume=false -Darguments="-DskipTests -Dspark=3.4 -Dscala=2.13" -Dspark=3.4 -Dscala=2.13 ``` -### Manually close and release the package - -1. Click `Close` on the Sedona staging repo on https://repository.apache.org under `staging repository` -2. Once the staging repo is closed, click `Release` on this repo. - - ## 9. Release Sedona Python and Zeppelin You must have the maintainer privilege of `https://pypi.org/project/apache-sedona/` and `https://www.npmjs.com/package/apache-sedona` @@ -450,8 +409,14 @@ Then submit to CRAN using this [web form](https://xmpalantir.wu.ac.at/cransubmit ### Generate Javadoc and Scaladoc -* Javadoc: Use Intelij IDEA to generate Javadoc for `core` and `viz` module, output them to `docs/api/javadoc` -* Scaladoc: Run `scaladoc -d docs/api/javadoc/sql/ sql/src/main/scala/org/apache/sedona/sql/utils/*.scala` +* Javadoc: Use Intellij IDEA to generate Javadoc for `core` and `viz` module, output them to `docs/api/javadoc`; or run the following script: +```bash +#!/bin/bash +mvn -q clean install -DskipTests +mv core/target/apidocs docs/api/javadoc/core +mv viz/target/apidocs docs/api/javadoc/viz +``` +* Scaladoc: Run `scaladoc -d docs/api/javadoc/sql/ sql/common/src/main/scala/org/apache/sedona/sql/utils/*.scala` Please do not commit these generated docs to Sedona GitHub. @@ -467,4 +432,4 @@ From [GitHub Action docs workflow](https://github.com/apache/sedona/actions/work 4. Check out the `gh-page` branch. 5. In a separate folder, check out GitHub sedona-website [asf-site branch](https://github.com/apache/sedona-website/tree/asf-site) 6. Copy all content to in Sedona main repo `gh-page` branch to Sedona website repo `asf-site` branch. -7. Commit and push the changes to the remote `asf-site` branch. \ No newline at end of file +7. Commit and push the changes to the remote `asf-site` branch. diff --git a/docs/community/snapshot.md b/docs/community/snapshot.md index 782bbd2c9b..8dbed0ce77 100644 --- a/docs/community/snapshot.md +++ b/docs/community/snapshot.md @@ -13,7 +13,7 @@ The detailed requirement is on [ASF Infra website](https://infra.apache.org/publ 1. In your local Sedona Git repo under master branch, run ```bash -echo " " > create-release.sh +echo '#!/bin/bash' > create-release.sh chmod 777 create-release.sh ``` 2. Use your favourite GUI text editor to open `create-release.sh`. @@ -28,29 +28,25 @@ In your Sedona GitHub repo, run this script: ```bash #!/bin/bash -source ~/.bashrc - git checkout master git pull rm -f release.* rm -f pom.xml.* +# Validate the POMs and your credential setup +mvn -q -B clean release:prepare -Dtag={{ sedona_create_release.current_git_tag }} -DreleaseVersion={{ sedona_create_release.current_version }} -DdevelopmentVersion={{ sedona_create_release.current_snapshot }} -Dresume=false -DdryRun=true -Penable-all-submodules -Darguments="-DskipTests" +mvn -q -B release:clean -Penable-all-submodules + # Spark 3.0 and Scala 2.12 -# Prepare the SNAPSHOTs -mvn -q -B clean -Darguments="-DskipTests" release:prepare -Dtag={{ sedona_create_release.current_git_tag }} -DreleaseVersion={{ sedona_create_release.current_version }} -DdevelopmentVersion={{ sedona_create_release.current_snapshot }} -DdryRun=true -DautoVersionSubmodules=true -Dresume=false -# Deploy the SNAPSHOTs -mvn -q deploy -DskipTests +mvn -q deploy -DskipTests -Dspark=3.0 -Dscala=2.12 -rm -f release.* -rm -f pom.xml.* +# Spark 3.0 and Scala 2.13 +mvn -q deploy -DskipTests -Dspark=3.0 -Dscala=2.13 -# Prepare for Spark 3.0 and Scala 2.13 -# Prepare the SNAPSHOTs -mvn -q -B clean -Darguments="-DskipTests -Dscala=2.13" release:prepare -Dtag={{ sedona_create_release.current_git_tag }} -DreleaseVersion={{ sedona_create_release.current_version }} -DdevelopmentVersion={{ sedona_create_release.current_snapshot }} -DdryRun=true -DautoVersionSubmodules=true -Dresume=false -# Deploy the SNAPSHOTs -mvn -q deploy -DskipTests -Dscala=2.13 +# Spark 3.4 and Scala 2.12 +mvn -q deploy -DskipTests -Dspark=3.4 -Dscala=2.12 -rm -f release.* -rm -f pom.xml.* -``` \ No newline at end of file +# Spark 3.4 and Scala 2.13 +mvn -q deploy -DskipTests -Dspark=3.4 -Dscala=2.13 +``` diff --git a/docs/setup/release-notes.md b/docs/setup/release-notes.md index fa607fc54b..38427dd4f3 100644 --- a/docs/setup/release-notes.md +++ b/docs/setup/release-notes.md @@ -4,6 +4,117 @@ !!!danger Sedona Python currently only works with Shapely 1.x. If you use GeoPandas, please use <= GeoPandas `0.11.1`. GeoPandas > 0.11.1 will automatically install Shapely 2.0. If you use Shapely, please use <= `1.8.4`. +## Sedona 1.4.1 + +Sedona 1.4.1 is compiled against, Spark 3.3 / Spark 3.4 / Flink 1.12, Java 8. + +### Highlights + +* [X] **Sedona Spark** More raster functions and bridge RasterUDT and Map Algebra operators. See [Raster based operators](../../api/sql/Raster-operators/#raster-based-operators) and [Raster to Map Algebra operators](../../api/sql/Raster-operators/#raster-to-map-algebra-operators). +* [X] **Sedona Spark & Flink** Added geodesic / geography functions: + * ST_DistanceSphere + * ST_DistanceSpheroid + * ST_AreaSpheroid + * ST_LengthSpheroid +* [X] **Sedona Spark & Flink** Introduced `SedonaContext` to unify Sedona entry points. +* [X] **Sedona Spark** Support Spark 3.4. +* [X] **Sedona Spark** Added a number of new ST functions. +* [X] **Zeppelin** Zeppelin helium plugin supports ploting geometries like linestring, polygon. + +### API change + +* **Sedona Spark & Flink** Introduced a new entry point called SedonaContext to unify all Sedona entry points in different compute engines and deprecate old Sedona register entry points. Users no longer have to register Sedona kryo serializer and import many tedious Python classes. + * **Sedona Spark**: + * Scala: + ```scala + import org.apache.sedona.spark.SedonaContext + val sedona = SedonaContext.create(SedonaContext.builder().master("local[*]").getOrCreate()) + sedona.sql("SELECT ST_GeomFromWKT(XXX) FROM") + ``` + * Python: + ```python + from sedona.spark import * + + config = SedonaContext.builder().\ + config('spark.jars.packages', + 'org.apache.sedona:sedona-spark-shaded-3.0_2.12:1.4.1,' + 'org.datasyslab:geotools-wrapper:1.4.0-28.2'). \ + getOrCreate() + sedona = SedonaContext.create(config) + sedona.sql("SELECT ST_GeomFromWKT(XXX) FROM") + ``` + * **Sedona Flink**: + ```java + import org.apache.sedona.flink.SedonaContext + StreamTableEnvironment sedona = SedonaContext.create(env, tableEnv); + sedona.sqlQuery("SELECT ST_GeomFromWKT(XXX) FROM") + ``` + +### Bug + +
    +
  • [SEDONA-266] - RS_Values throws UnsupportedOperationException for shuffled point arrays +
    • +
  • [SEDONA-267] - Cannot pip install apache-sedona 1.4.0 from source distribution +
    • +
  • [SEDONA-273] - Set a upper bound for Shapely, Pandas and GeoPandas +
    • +
  • [SEDONA-277] - Sedona spark artifacts for scala 2.13 do not have proper POMs +
    • +
  • [SEDONA-283] - Artifacts were deployed twice when running mvn clean deploy +
    • +
  • [SEDONA-284] - Property values in dependency deduced POMs for shaded modules were not substituted +
    • +
+ +### New Feature + +
    +
  • [SEDONA-196] - Add ST_Force3D to Sedona +
    • +
  • [SEDONA-239] - Implement ST_NumPoints +
    • +
  • [SEDONA-264] - zeppelin helium plugin supports ploting geometry like linestring, polygon +
    • +
  • [SEDONA-280] - Add ST_GeometricMedian +
    • +
  • [SEDONA-281] - Support geodesic / geography functions +
    • +
  • [SEDONA-286] - Support optimized distance join on ST_DistanceSpheroid and ST_DistanceSphere +
    • +
  • [SEDONA-287] - Use SedonaContext to unify Sedona entry points +
    • +
  • [SEDONA-292] - Bridge Sedona Raster and Map Algebra operators +
    • +
+ +### Improvement + +
    +
  • [SEDONA-167] - Add __pycache__ to Python .gitignore +
    • +
  • [SEDONA-265] - Migrate all ST functions to Sedona Inferred Expressions +
    • +
  • [SEDONA-269] - Add data source for writing binary files +
    • +
  • [SEDONA-270] - Remove redundant serialization for rasters +
    • +
  • [SEDONA-271] - Add raster function RS_SRID +
    • +
  • [SEDONA-274] - Move all ST function logics to Sedona common +
    • +
  • [SEDONA-275] - Add raster function RS_SetSRID +
    • +
  • [SEDONA-276] - Add support for Spark 3.4 +
    • +
  • [SEDONA-279] - Sedona-Flink should not depend on Sedona-Spark modules +
    • +
  • [SEDONA-282] - R – Add raster write function +
    • +
  • [SEDONA-290] - RDD Spatial Joins should follow the iterator model +
    • +
+ ## Sedona 1.4.0 Sedona 1.4.0 is compiled against, Spark 3.3 / Flink 1.12, Java 8. diff --git a/mkdocs.yml b/mkdocs.yml index a9ec3645d5..883c3a8e0e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -131,11 +131,11 @@ extra: current_version: 1.4.0 current_geotools: 1.4.0-28.2 sedona_create_release: - current_version: 1.4.0 - current_git_tag: sedona-1.4.0-rc1 - current_rc: 1.4.0-rc1 - current_snapshot: 1.4.1-SNAPSHOT - next_version: 1.4.1 + current_version: 1.4.1 + current_git_tag: sedona-1.4.1-rc1 + current_rc: 1.4.1-rc1 + current_snapshot: 1.5.0-SNAPSHOT + next_version: 1.5.0 copyright: Copyright © 2023 The Apache Software Foundation markdown_extensions: - admonition From 04e3642e3bb8ed1a4df553295b6ef160214b6c0e Mon Sep 17 00:00:00 2001 From: Kristin Cowalcijk Date: Tue, 20 Jun 2023 20:42:51 +0800 Subject: [PATCH 7/7] Update docs for generating Javadoc and Scaladoc --- docs/community/publish.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/community/publish.md b/docs/community/publish.md index bf524643a4..07cc37f820 100644 --- a/docs/community/publish.md +++ b/docs/community/publish.md @@ -409,14 +409,16 @@ Then submit to CRAN using this [web form](https://xmpalantir.wu.ac.at/cransubmit ### Generate Javadoc and Scaladoc -* Javadoc: Use Intellij IDEA to generate Javadoc for `core` and `viz` module, output them to `docs/api/javadoc`; or run the following script: +Run the following script to build Javadoc and Scaladoc of sedona modules and move them to docs/api/javadoc directory. + ```bash #!/bin/bash + mvn -q clean install -DskipTests mv core/target/apidocs docs/api/javadoc/core mv viz/target/apidocs docs/api/javadoc/viz +mv sql/common/target/site/scaladocs docs/api/javadoc/sql ``` -* Scaladoc: Run `scaladoc -d docs/api/javadoc/sql/ sql/common/src/main/scala/org/apache/sedona/sql/utils/*.scala` Please do not commit these generated docs to Sedona GitHub.