Merge branch 'master' of github.com:lablup/backend.ai-kernels

vendor/ngc-pytorch/requirements.22.05.txt

@@ -261,7 +261,7 @@ sacrebleu==1.3.6
 sacred==0.8.1
 schema==0.7.1
 scikit-image==0.19.0
-cikit-optimize==0.9.0
+scikit-optimize==0.9.0
 scikit-plot==0.3.7
 scikit-surprise==1.1.1
 scooby==0.5.5

vendor/ngc-tensorflow/Dockerfile.22.09-tf2-py3

@@ -0,0 +1,255 @@
+FROM nvcr.io/nvidia/tensorflow:22.09-tf2-py3
+# NVIDIA DIGITS runs on Python 3.8
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    MPLBACKEND=Svg \
+    PIP_IGNORE_INSTALLED=0 \
+    PYTHONUNBUFFERED=1 \
+    LD_LIBRARY_PATH="/usr/local/cuda/compat/lib:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/nvidia/lib64:/usr/include/x86_64-linux-gnu" \
+    PATH="/usr/local/nvm/versions/node/v16.15.1/bin:/usr/local/mpi/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/ucx/bin:/opt/tensorrt/bin:/usr/local/src/lightgbm/LightGBM:/usr/local/bin/mecab" \
+    mecab_dicdir=/usr/local/lib/mecab/dic/mecab-ko-dic \
+    CPLUS_INCLUDE_PATH=/usr/include/gdal \
+    C_INCLUDE_PATH=/usr/include/gdal \
+    LANG=C.UTF-8
+
+RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - && \
+    ln -s /usr/include/libv4l1-videodev.h /usr/include/linux/videodev.h && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends \
+	automake \
+	default-libmysqlclient-dev \
+	ffmpeg \
+	fonts-nanum \
+	fonts-nanum-coding \
+	fonts-nanum-extra \
+	gfortran \
+	gir1.2-gtk-3.0 \
+	htop \
+	libaio1 \
+	libasound2-dev \
+	libatlas-base-dev \
+	libavformat-dev \
+	libavresample-dev \
+	libcairo2-dev \
+	libdc1394-22-dev \
+	libeigen3-dev \
+	libfaac-dev \
+	libgdal-dev \
+	libgflags-dev \
+	libgoogle-glog-dev \
+	libgirepository1.0-dev \
+	libgphoto2-dev \
+	libgstreamer-plugins-base1.0-dev \
+	libgstreamer1.0-dev \
+	libgtk-3-dev \
+	libhdf5-dev \	
+	libmp3lame-dev \
+	libopenblas-dev \
+	libopencore-amrnb-dev \
+	libopencore-amrwb-dev \
+	libportmidi-dev \
+	libprotobuf-dev \
+	libsdl-dev \
+	libsdl-image1.2-dev \
+	libsdl-mixer1.2-dev \
+	libsdl-ttf2.0-dev \
+	libsm6 \
+	libswscale-dev \
+	libtbb-dev \
+	libtheora-dev \
+	libvorbis-dev \
+	libx264-dev \
+	libxext6 \
+	libxrender-dev \
+	libxvidcore-dev \
+	mercurial \
+	libncurses-dev \
+	ncurses-term \
+	nodejs \
+	pdsh \
+	protobuf-compiler \
+        libsmpeg-dev \
+        openjdk-8-jdk \
+        python3-gdal \
+	v4l-utils \	
+	x264 
+
+RUN mkdir -p /opt/oracle && \
+    cd /opt/oracle && \
+    wget https://download.oracle.com/otn_software/linux/instantclient/instantclient-basiclite-linuxx64.zip && \
+    unzip instantclient-basiclite-linuxx64.zip && \
+    rm -f instantclient-basiclite-linuxx64.zip && \
+    cd /opt/oracle/instantclient* && \
+    rm -f *jdbc* *occi* *mysql* *README *jar uidrvci genezi adrci && \
+    echo /opt/oracle/instantclient* > /etc/ld.so.conf.d/oracle-instantclient.conf && \
+    ldconfig
+
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 2 && \
+    python3 -m pip install -U "pip < 22" setuptools wheel
+
+# install NLP packages *mecab-ko & khai*
+WORKDIR /tmp
+RUN curl -LO https://bitbucket.org/eunjeon/mecab-ko/downloads/mecab-0.996-ko-0.9.2.tar.gz && \
+    tar zxfv mecab-0.996-ko-0.9.2.tar.gz && \
+    cd mecab-0.996-ko-0.9.2 && \
+    ./configure && \
+    make -j$(nproc) && \
+    make check && \
+    make install && \
+    echo "Install mecab-ko-dic" && \
+    cd /tmp && \
+    ldconfig && \
+    curl -LO https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/mecab-ko-dic-2.1.1-20180720.tar.gz && \
+    tar -zxvf mecab-ko-dic-2.1.1-20180720.tar.gz && \
+    cd mecab-ko-dic-2.1.1-20180720 && \
+    ./autogen.sh && \
+    ./configure && \
+    make -j$(nproc) && \
+    sh -c 'echo "dicdir=/usr/local/lib/mecab/dic/mecab-ko-dic" > /usr/local/etc/mecabrc' && \
+    make install && \
+    cd /tmp && \
+    git clone https://bitbucket.org/eunjeon/mecab-python-0.996.git && \
+    python3 -m pip install /tmp/mecab-python-0.996
+
+# OpenCV
+ENV OPENCV_VERSION="4.5.5"
+WORKDIR /tmp
+RUN wget https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip && \
+    wget -O opencv-contrib.zip https://github.com/opencv/opencv_contrib/archive/${OPENCV_VERSION}.zip && \
+    unzip ${OPENCV_VERSION}.zip && \
+    unzip opencv-contrib.zip && \
+    mkdir opencv-${OPENCV_VERSION}/cmake_binary && \
+    cd opencv-${OPENCV_VERSION}/cmake_binary && \
+    cmake \
+      -DCMAKE_BUILD_TYPE=RELEASE \
+      -D BUILD_TIFF=ON \
+      -D BUILD_opencv_java=OFF \
+      -D WITH_CUDA=ON \
+      -D CUDA_NVCC_FLAGS=--expt-relaxed-constexpr \
+      -D CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-11.8 \
+      -D ENABLE_FAST_MATH=1 \
+      -D CUDA_FAST_MATH=1 \
+      -D WITH_CUBLAS=1 \
+      -D WITH_OPENGL=ON \
+      -D WITH_OPENCL=ON \
+      -D WITH_IPP=ON \
+      -D WITH_TBB=ON \
+      -D WITH_EIGEN=ON \
+      -D WITH_V4L=ON \
+      -D BUILD_TESTS=OFF \
+      -D BUILD_PERF_TESTS=OFF \
+      -D OPENCV_EXTRA_MODULES_PATH="../../opencv_contrib-4.5.5/modules" \
+      -D CMAKE_BUILD_TYPE=RELEASE \
+      -D CMAKE_INSTALL_PREFIX=$(python3 -c "import sys; print(sys.prefix)") \
+      -D PYTHON_EXECUTABLE=$(which python3) \
+      -D PYTHON_INCLUDE_DIR=$(python3 -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") \
+      -D PYTHON_PACKAGES_PATH=$(python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") \
+      .. 2>&1 | tee cmake_messages.txt && \
+    make -j$(nproc) && \
+    make install && \
+    cd /tmp && \
+    python3 -m pip install --no-cache-dir opencv-python && \
+    rm -fr opencv* 
+
+WORKDIR /tmp
+RUN git clone -q --branch=v0.3.20 https://github.com/xianyi/OpenBLAS.git && \
+    cd OpenBLAS && \
+    make DYNAMIC_ARCH=1 NO_AFFINITY=1 NUM_THREADS=48 FC=gfortran && \
+    make install
+RUN git clone --recursive https://github.com/bodono/scs-python.git  && \
+    cd /tmp/scs-python && \
+    python setup.py install --scs --gpu
+
+WORKDIR /tmp
+COPY ./requirements.22.09.txt /tmp/requirements.txt
+RUN python3 -m pip install --no-cache-dir -r requirements.txt && \
+    rm -f /tmp/*.whl /tmp/requirements.txt
+
+# install bashtop
+WORKDIR /tmp
+RUN git clone https://github.com/aristocratos/bashtop.git && \
+    cd bashtop && \
+    make install
+
+# install git-lfs
+WORKDIR /tmp
+RUN curl -sLO https://github.com/git-lfs/git-lfs/releases/download/v3.2.0/git-lfs-linux-amd64-v3.2.0.tar.gz && \
+    tar -zxf git-lfs-linux-amd64-v3.2.0.tar.gz && \
+    cd git-lfs-3.2.0 && \
+    bash install.sh && \
+    rm -rf /tmp/*
+
+RUN curl -fL https://github.com/cdr/code-server/releases/download/v4.5.1/code-server-4.5.1-linux-amd64.tar.gz | tar -C /usr/local/lib -xz && \
+    mv /usr/local/lib/code-server-4.5.1-linux-amd64 /usr/local/lib/code-server-4.5.1 && \
+    ln -s /usr/local/lib/code-server-4.5.1/bin/code-server /usr/local/bin/code-server
+
+# Install Open MPI
+RUN mkdir /tmp/openmpi && \
+    cd /tmp/openmpi && \
+    wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.4.tar.gz && \
+    tar zxf openmpi-4.1.4.tar.gz && \
+    cd openmpi-4.1.4 && \
+    ./configure --enable-orterun-prefix-by-default && \
+    make -j $(nproc) all && \
+    make install && \
+    ldconfig && \
+    rm -rf /tmp/openmpi*
+# Create a wrapper for OpenMPI to allow running as root by default
+RUN mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real && \
+    echo '#!/bin/bash' > /usr/local/bin/mpirun && \
+    echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/local/bin/mpirun && \
+    chmod a+x /usr/local/bin/mpirun
+
+# Configure OpenMPI to run good defaults:
+RUN echo "btl_tcp_if_exclude = lo,docker0" >> /usr/local/etc/openmpi-mca-params.conf
+
+RUN python3 -m pip install --no-cache-dir \
+    	    mpi4py==3.1.3 \
+	    nni==2.8 \
+	    mlflow==1.27.0 \
+	    scikit-nni==0.2.1
+
+# Install Jupyterlab extensions
+RUN jupyter nbextensions_configurator enable && \
+    jupyter contrib nbextension install && \
+    jupyter nbextension enable --py --sys-prefix widgetsnbextension && \
+    jupyter serverextension enable --py jupyterlab --sys-prefix && \
+    jupyter labextension install --no-build @jupyter-widgets/jupyterlab-manager && \
+    jupyter labextension install --no-build @krassowski/jupyterlab-lsp && \
+    jupyter serverextension enable --py jupyter_lsp && \
+    jupyter labextension install --no-build @jupyterlab/toc && \
+    jupyter nbextension enable execute_time/ExecuteTime && \
+    jupyter nbextension enable toc2/main && \
+    jupyter labextension install @jupyterlab/toc && \
+    jupyter lab build    
+
+RUN apt autoclean && \
+    sed -i 's/source \/usr\/local\/nvm\/nvm.sh//' /etc/bash.bashrc && \
+    ln -sf /usr/share/terminfo/x/xterm-color /usr/share/terminfo/x/xterm-256color && \
+    rm -rf /var/lib/apt/lists/* && \	
+    rm -rf /root/.cache && \
+    rm -rf /tmp/*
+
+# Install ipython kernelspec
+RUN /usr/bin/python3 -m ipykernel install --display-name "Python 3.8 (NGC 22.09 / TensorFlow 2.9.1) on Backend.AI" && \
+    cat /usr/local/share/jupyter/kernels/python3/kernel.json
+
+# Backend.AI specifics
+COPY ./service-defs /etc/backend.ai/service-defs
+COPY ./runner-scripts/bootstrap.sh runner-scripts/setup_multinode.py /opt/container/
+LABEL ai.backend.kernelspec="1" \
+      ai.backend.envs.corecount="OPENBLAS_NUM_THREADS,OMP_NUM_THREADS,NPROC" \
+      ai.backend.features="batch query uid-match user-input" \
+      ai.backend.base-distro="ubuntu16.04" \
+      ai.backend.accelerators="cuda" \
+      ai.backend.resource.min.cpu="1" \
+      ai.backend.resource.min.mem="1g" \
+      ai.backend.resource.min.cuda.device=0 \
+      ai.backend.resource.min.cuda.shares=0 \
+      ai.backend.base-distro="ubuntu16.04" \
+      ai.backend.runtime-type="python" \
+      ai.backend.runtime-path="/usr/bin/python" \
+      ai.backend.service-ports="ipython:pty:3000,jupyter:http:8091,jupyterlab:http:8090,vscode:http:8180,tensorboard:http:6006,mlflow-ui:preopen:5000,nniboard:preopen:8080"
+
+WORKDIR /home/work
+# vim: ft=dockerfile