apache · timyuer · May 7, 2024 · May 25, 2024
diff --git a/bigtop-packages/src/common/doris/do-component-build b/bigtop-packages/src/common/doris/do-component-build
@@ -0,0 +1,27 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+#load versions
+. `dirname $0`/bigtop.bom
+
+export DORIS_BUILD_PYTHON_VERSION="python3"
+export DORIS_TOOLCHAIN="clang"
+export REPOSITORY_URL="https://doris-thirdparty-repo.bj.bcebos.com/thirdparty"
+export CUSTOM_NPM_REGISTRY="https://registry.npmmirror.com"
+
+sh build.sh --fe --be --clean
diff --git a/bigtop-packages/src/common/doris/install_doris.sh b/bigtop-packages/src/common/doris/install_doris.sh
@@ -0,0 +1,158 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+usage() {
+  echo "
+usage: $0 <options>
+  Required not-so-options:
+     --build-dir=DIR                      path to doris dist.dir
+     --source-dir=DIR                     path to package shared files dir
+     --prefix=PREFIX                      path to install into
+
+  Optional options:
+     --doris-fe-lib-dir=DIR               path to install doris home [/usr/lib/doris-fe]
+     --doris-be-lib-dir=DIR               path to install doris home [/usr/lib/doris-be]
+     --bin-dir=DIR                        path to install bins [/usr/bin]
+     --lib-hadoop=DIR                     path to hadoop home [/usr/lib/hadoop]
+     --etc-doris=DIR                   path to install doris conf [/etc/doris]
+     ... [ see source for more similar options ]
+  "
+  exit 1
+}
+
+OPTS=$(getopt \
+  -n $0 \
+  -o '' \
+  -l 'prefix:' \
+  -l 'doris-fe-lib-dir:' \
+  -l 'doris-be-lib-dir:' \
+  -l 'bin-dir:' \
+  -l 'lib-hadoop:' \
+  -l 'etc-doris:' \
+  -l 'source-dir:' \
+  -l 'build-dir:' -- "$@")
+
+if [ $? != 0 ] ; then
+    usage
+fi
+
+eval set -- "$OPTS"
+while true ; do
+    case "$1" in
+        --prefix)
+        PREFIX=$2 ; shift 2
+        ;;
+        --build-dir)
+        BUILD_DIR=$2 ; shift 2
+        ;;
+        --source-dir)
+        SOURCE_DIR=$2 ; shift 2
+        ;;
+        --doris-fe-lib-dir)
+        DORIS_FE_LIB_DIR=$2 ; shift 2
+        ;;
+        --doris-be-lib-dir)
+        DORIS_BE_LIB_DIR=$2 ; shift 2
+        ;;
+        --bin-dir)
+        BIN_DIR=$2 ; shift 2
+        ;;
+        --lib-hadoop)
+        LIB_HADOOP=$2 ; shift 2
+        ;;
+        --etc-doris)
+        ETC_DORIS=$2 ; shift 2
+        ;;
+        --)
+        shift ; break
+        ;;
+        *)
+        echo "Unknown option: $1"
+        usage
+        exit 1
+        ;;
+    esac
+done
+
+for var in PREFIX BUILD_DIR SOURCE_DIR ; do
+  if [ -z "$(eval "echo \$$var")" ]; then
+    echo Missing param: $var
+    usage
+  fi
+done
+
+# load bigtop component versions
+if [ -f "$SOURCE_DIR/bigtop.bom" ]; then
+  . $SOURCE_DIR/bigtop.bom
+fi
+
+
+BIN_DIR=${BIN_DIR:-/usr/bin}
+LIB_HADOOP=${LIB_HADOOP:-/usr/lib/hadoop}
+DORIS_FE_LIB_DIR=${DORIS_FE_LIB_DIR:-/usr/lib/doris-fe}
+DORIS_BE_LIB_DIR=${DORIS_BE_LIB_DIR:-/usr/lib/doris-be}
+
+ETC_DORIS=${ETC_DORIS:-/etc/doris}
+ETC_DORIS_FE=$ETC_DORIS/conf.dist/doris-fe
+ETC_DORIS_BE=$ETC_DORIS/conf.dist/doris-be
+
+# No prefix
+NP_ETC_DORIS=/etc/doris
+NP_ETC_DORIS_FE=$NP_ETC_DORIS/conf.dist/doris-fe
+NP_ETC_DORIS_BE=$NP_ETC_DORIS/conf.dist/doris-be
+NP_VAR_LOG_DORIS_FE=/var/log/doris-fe
+NP_VAR_LOG_DORIS_BE=/var/log/doris-be
+NP_ETC_HADOOP=/etc/hadoop
+
+install -d -m 0755 $PREFIX/$NP_ETC_DORIS
+# Doris FE
+install -d -m 0755 $PREFIX/$DORIS_FE_LIB_DIR
+install -d -m 0755 $PREFIX/$NP_ETC_DORIS_FE
+install -d -m 0755 $PREFIX/$ETC_DORIS_FE
+install -d -m 0755 $PREFIX/$NP_VAR_LOG_DORIS_FE
+install -d -m 0755 $PREFIX/var/run/doris-fe
+# Doris BE
+install -d -m 0755 $PREFIX/$DORIS_BE_LIB_DIR
+install -d -m 0755 $PREFIX/$NP_ETC_DORIS_BE
+install -d -m 0755 $PREFIX/$ETC_DORIS_BE
+install -d -m 0755 $PREFIX/$NP_VAR_LOG_DORIS_BE
+install -d -m 0755 $PREFIX/var/run/doris-be
+
+cp -ra ${BUILD_DIR}/fe/* $PREFIX/${DORIS_FE_LIB_DIR}/
+cp -ra ${BUILD_DIR}/be/* $PREFIX/${DORIS_BE_LIB_DIR}/
+
+# remove conf directory
+rm -rf $PREFIX/${LIB_DIR}/$DORIS_FE_LIB_DIR/conf
+rm -rf $PREFIX/${LIB_DIR}/$DORIS_BE_LIB_DIR/conf
+# remove log directory
+rm -rf $PREFIX/${LIB_DIR}/$DORIS_FE_LIB_DIR/log
+rm -rf $PREFIX/${LIB_DIR}/$DORIS_BE_LIB_DIR/log
+
+# Copy the configuration files
+cp -ra ${BUILD_DIR}/fe/conf/* $PREFIX/$ETC_DORIS_FE
+cp -ra ${BUILD_DIR}/be/conf/* $PREFIX/$ETC_DORIS_BE
+
+# link the conf directory
+ln -s $NP_ETC_DORIS_FE $PREFIX/$DORIS_FE_LIB_DIR/conf
+ln -s $NP_ETC_DORIS_BE $PREFIX/$DORIS_BE_LIB_DIR/conf
+# link the log directory
+ln -s $NP_VAR_LOG_DORIS_FE $PREFIX/$DORIS_FE_LIB_DIR/log
+ln -s $NP_VAR_LOG_DORIS_FE $PREFIX/$DORIS_BE_LIB_DIR/log
+
+
diff --git a/bigtop-packages/src/common/doris/patch0-DORIS-23982.diff b/bigtop-packages/src/common/doris/patch0-DORIS-23982.diff
@@ -0,0 +1,23 @@
+diff --git a/build.sh b/build.sh
+index 43ae8d8d2e..7ab0473715 100755
+--- a/build.sh
++++ b/build.sh
+@@ -290,10 +290,16 @@ update_submodule() {
+     fi
+     set -e
+     if [[ "${exit_code}" -ne 0 ]]; then
++        set +e
+         # try to get submodule's current commit
+         submodule_commit=$(git ls-tree HEAD "${submodule_path}" | awk '{print $3}')
+-
+-        commit_specific_url=$(echo "${archive_url}" | sed "s/refs\/heads/${submodule_commit}/")
++        exit_code=$?
++        if [[ "${exit_code}" = "0" ]]; then
++            commit_specific_url=$(echo "${archive_url}" | sed "s/refs\/heads/${submodule_commit}/")
++        else
++            commit_specific_url="${archive_url}"
++        fi
++        set -e
+         echo "Update ${submodule_name} submodule failed, start to download and extract ${commit_specific_url}"
+
+         mkdir -p "${DORIS_HOME}/${submodule_path}"
diff --git a/bigtop-packages/src/rpm/doris/SPECS/doris.spec b/bigtop-packages/src/rpm/doris/SPECS/doris.spec
@@ -0,0 +1,156 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+%define doris_name doris
+%define doris_fe_name doris-fe
+%define doris_be_name doris-be
+%define doris_pkg_name %{doris_name}%{pkg_name_suffix}
+
+%define etc_default %{parent_dir}/etc/default
+
+%define usr_lib_doris_fe %{parent_dir}/usr/lib/%{doris_fe_name}
+%define usr_lib_doris_be %{parent_dir}/usr/lib/%{doris_be_name}
+%define var_lib_doris_fe %{parent_dir}/var/lib/%{doris_fe_name}
+%define var_lib_doris_be %{parent_dir}/var/lib/%{doris_be_name}
+%define etc_doris %{parent_dir}/etc/%{doris_name}
+
+%define usr_lib_hadoop %{parent_dir}/usr/lib/hadoop
+
+%define bin_dir %{parent_dir}/%{_bindir}
+%define man_dir %{parent_dir}/%{_mandir}
+%define doc_dir %{parent_dir}/%{_docdir}
+
+# No prefix directory
+%define np_var_log_doris_fe /var/log/%{doris_fe_name}
+%define np_var_log_doris_be /var/log/%{doris_be_name}
+%define np_var_run_doris_fe /var/run/%{doris_fe_name}
+%define np_var_run_doris_be /var/run/%{doris_be_name}
+%define np_etc_doris /etc/%{doris_name}
+
+%define doris_services fe be
+%define build_target_doris output
+
+%global __python %{__python3}
+
+
+%if  %{!?suse_version:1}0
+%define doc_doris %{doc_dir}/%{doris_name}-%{doris_version}
+%define alternatives_cmd alternatives
+%global initd_dir %{_sysconfdir}/rc.d/init.d
+%else
+%define doc_doris %{doc_dir}/%{doris_name}-%{doris_version}
+%define alternatives_cmd update-alternatives
+%global initd_dir %{_sysconfdir}/rc.d
+%endif
+
+Name: %{doris_pkg_name}
+Version: %{doris_version}
+Release: %{doris_release}
+Summary: Apache Doris is an easy-to-use, high performance and unified analytics database.
+License: ASL 2.0
+URL: http://doris.apache.org/
+Group: Development/Libraries
+Buildroot: %{_topdir}/INSTALL/%{name}-%{version}
+Source0: %{doris_name}-%{doris_base_version}.tar.gz
+Source1: do-component-build
+Source2: install_doris.sh
+Source3: init.d.tmpl
+Source4: bigtop.bom
+#BIGTOP_PATCH_FILES
+Requires: bigtop-utils >= 0.7
+Requires(preun): /sbin/service
+Requires: python3
+
+%description
+Apache Doris is an easy-to-use, high performance and unified analytics database.
+
+%package fe
+Summary: Provides the Apache Doris FE service.
+Group: System/Daemons
+Requires: %{name} = %{version}-%{release}
+Requires(pre): %{name} = %{version}-%{release}
+
+%description fe
+Apache Doris FE service.
+
+%package be
+Summary: Provides the Apache Doris BE service.
+Group: System/Daemons
+Requires: %{name} = %{version}-%{release}
+Requires(pre): %{name} = %{version}-%{release}
+
+%description be
+Apache Doris BE service.
+
+##############################################
+
+%prep
+%setup -n apache-%{doris_name}-%{doris_base_version}-src
+#BIGTOP_PATCH_COMMANDS
+
+%build
+bash $RPM_SOURCE_DIR/do-component-build
+
+
+
+# Init.d scripts
+%__install -d -m 0755 $RPM_BUILD_ROOT/%{initd_dir}/
+
+%install
+%__rm -rf $RPM_BUILD_ROOT
+
+sh -x %{SOURCE2} \
+    --prefix=$RPM_BUILD_ROOT \
+    --source-dir=$RPM_SOURCE_DIR \
+    --build-dir=`pwd`/%{build_target_doris} \
+    --doris-fe-lib-dir=%{usr_lib_doris_fe} \
+    --doris-be-lib-dir=%{usr_lib_doris_be} \
+    --bin-dir=%{bin_dir} \
+    --lib-hadoop=%{usr_lib_hadoop} \
+    --etc-doris=%{etc_doris}
+
+%pre
+getent group doris >/dev/null || groupadd -r doris
+getent passwd doris >/dev/null || useradd -c "Doris" -s /sbin/nologin -g doris -r -d %{usr_lib_doris_fe} doris 2> /dev/null || :
+
+%post
+%{alternatives_cmd} --install %{np_etc_doris}/conf %{doris_name}-conf %{etc_doris}/conf.dist 30
+
+%preun
+if [ "$1" = 0 ]; then
+        %{alternatives_cmd} --remove %{doris_fe_name}-conf %{etc_doris}/conf.dist || :
+fi
+
+###### FILES ###########
+
+%files
+%defattr(-,root,root,755)
+%attr(0755,doris,doris) %{np_etc_doris}
+%config(noreplace) %{etc_doris}/conf.dist
+
+%files fe
+%defattr(-,root,root,755)
+%config(noreplace) %{etc_doris}/conf.dist/doris-fe
+%attr(0755,doris,doris) %{np_var_log_doris_fe}
+%attr(0755,doris,doris) %{np_var_run_doris_fe}
+%{usr_lib_doris_fe}
+
+%files be
+%defattr(-,root,root,755)
+%config(noreplace) %{etc_doris}/conf.dist/doris-be
+%attr(0755,doris,doris) %{np_var_log_doris_be}
+%attr(0755,doris,doris) %{np_var_run_doris_be}
+%{usr_lib_doris_be}
+
diff --git a/bigtop.bom b/bigtop.bom
@@ -354,5 +354,15 @@ bigtop {
                 archive = site }
       maven_parallel_build = true
     }
+    'doris' {
+      name    = 'doris'
+      relNotes = 'Apache Doris'
+      version { base = '2.0.9'; pkg = base; release = 1 }
+      tarball { destination = "$name-${version.base}.tar.gz"
+                source      = "apache-$name-${version.base}-src.tar.gz" }
+      url     { download_path =  String.format("%s/%s/%s/", name, version.base.substring(0, version.base.lastIndexOf(".")), version.base)
+                site = "${apache.APACHE_MIRROR}/${download_path}"
+                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
+    }
   }
 }
diff --git a/bigtop_toolchain/manifests/installer.pp b/bigtop_toolchain/manifests/installer.pp
@@ -19,6 +19,7 @@
   include bigtop_toolchain::maven
   include bigtop_toolchain::ant
   include bigtop_toolchain::gradle
+  include bigtop_toolchain::node
   include bigtop_toolchain::protobuf
   include bigtop_toolchain::packages
   include bigtop_toolchain::env