Add support for Habana 1.17.1

- Add structure for Habana 1.17.1 image
- Fix reference to Python 3.10
- Add Habana 1.17.1 Python 3.11 base image to Makefile
- Fix references in README files from older Habana images
- Fix references to Habana official documentation
- Fix README for Habana 1.17.1
- Fix Habana Vault container image reference
- Fix Habana 1.17.1's Pipfile and Pipfile.lock files
- Add Habana 1.17.1 to Makefile's refresh-pipfilelock-files function
- Change entrypoint's chmod for Habana 1.17.1
- Fix Makefile call for Habana 1.17.1
- Add missing newline at end of Dockerfile
- Add Habana 1.17.1 images to Github Actions matrix jobs
- Add instructions to run the build and push commands for Habana 1.17.1

.github/workflows/build-notebooks.yaml

@@ -603,6 +603,26 @@
             },
             "secrets": "inherit"
         },
+        "base-habana-python-3_11": {
+            "needs": [],
+            "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
+            "with": {
+                "target": "base-habana-python-3.11",
+                "github": "${{ toJSON(github) }}"
+            },
+            "secrets": "inherit"
+        },
+        "habana-jupyter-1_17_1-ubi9-python-3_11": {
+            "needs": [
+                "base-habana-python-3_11"
+            ],
+            "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
+            "with": {
+                "target": "habana-jupyter-1.17.1-ubi9-python-3.11",
+                "github": "${{ toJSON(github) }}"
+            },
+            "secrets": "inherit"
+        },
         "base-c9s-python-3_9": {
             "needs": [],
             "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",

Makefile

@@ -352,6 +352,16 @@ intel-runtime-ml-ubi9-python-3.11: base-ubi9-python-3.11
 jupyter-intel-ml-ubi9-python-3.11: base-ubi9-python-3.11
 	$(call image,$@,jupyter/intel/ml/ubi9-python-3.11,$<)
 
+# Build and push base-habana-python-3.11 image to the registry
+.PHONY: base-habana-python-3.11
+base-habana-python-3.11:
+	$(call image,$@,base/habana-python-3.11)
+
+# Build and push habana-1.17.1-ubi9-python-3.11 image to the registry
+.PHONY: habana-jupyter-1.17.1-ubi9-python-3.11
+habana-jupyter-1.17.1-ubi9-python-3.11: base-habana-python-3.11
+	$(call image,$@,habana/1.17.1/ubi9-python-3.11,$<)
+
 ####################################### Buildchain for Python 3.9 using C9S #######################################
 
 # Build and push base-c9s-python-3.9 image to the registry
@@ -741,6 +751,7 @@ refresh-pipfilelock-files:
 	cd base/ubi8-python-3.8 && pipenv lock
 	cd base/ubi9-python-3.9 && pipenv lock
 	cd base/c9s-python-3.9 && pipenv lock
+	cd base/habana-python-3.11 && pipenv lock
 	cd jupyter/minimal/ubi8-python-3.8 && pipenv lock
 	cd jupyter/minimal/ubi9-python-3.9 && pipenv lock
 	cd jupyter/datascience/ubi8-python-3.8 && pipenv lock
@@ -750,6 +761,7 @@ refresh-pipfilelock-files:
 	cd jupyter/trustyai/ubi9-python-3.9 && pipenv lock
 	cd habana/1.10.0/ubi8-python-3.8 && pipenv lock
 	cd habana/1.13.0/ubi8-python-3.8 && pipenv lock
+	cd habana/1.17.1/ubi9-python-3.11 && pipenv lock
 	cd runtimes/minimal/ubi8-python-3.8 && pipenv lock
 	cd runtimes/minimal/ubi9-python-3.9 && pipenv lock
 	cd runtimes/datascience/ubi8-python-3.8 && pipenv lock

base/habana-python-3.11/Dockerfile

@@ -0,0 +1,82 @@
+# Reference:
+#  - https://vault.habana.ai/ui/repos/tree/General/gaudi-docker/1.17.1/rhel9.4/habanalabs/pytorch-installer-2.3.1/1.17.1-40
+FROM vault.habana.ai/gaudi-docker/1.17.1/rhel9.4/habanalabs/pytorch-installer-2.3.1:1.17.1-40 as base
+
+LABEL name="odh-notebook-habana-jupyter-1.17.1-ubi9-python-3.11" \
+    summary="Jupyter HabanaAI 1.17.1 notebook image for ODH notebooks" \
+    description="Jupyter HabanaAI 1.17.1 notebook image with base Python 3.11 builder image based on ubi9 for ODH notebooks" \
+    io.k8s.display-name="Jupyter HabanaAI 1.17.1 notebook image for ODH notebooks" \
+    io.k8s.description="Jupyter HabanaAI 1.17.1 notebook image with base Python 3.11 builder image based on ubi9 for ODH notebooks" \
+    authoritative-source-url="https://github.com/opendatahub-io/notebooks" \
+    io.openshift.build.commit.ref="main" \
+    io.openshift.build.source-location="https://github.com/opendatahub-io/notebooks/tree/main/habana/1.17.1/ubi9-python-3.11" \
+    io.openshift.build.image="quay.io/opendatahub/workbench-images:habana-jupyter-1.17.1-ubi9-python-3.11"
+
+COPY --from=registry.access.redhat.com/ubi9/python-39:latest /usr/bin/fix-permissions /usr/bin
+COPY --from=registry.access.redhat.com/ubi9/python-39:latest /usr/bin/rpm-file-permissions /usr/bin
+
+ENV PATH=/opt/app-root/src/bin:/opt/app-root/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
+    APP_ROOT="/opt/app-root"
+
+ENV HOME="${APP_ROOT}/src"
+
+WORKDIR ${HOME}
+
+RUN mkdir -p ${APP_ROOT}/src && \
+    rpm-file-permissions && \
+    useradd -u 1001 -r -g 0 -d ${HOME} -c "Default Application User" default && \
+    chown -R 1001:0 ${APP_ROOT}
+
+FROM base as notebook-base
+
+ENV PYTHON_VERSION=3.11 \
+    PATH=$HOME/.local/bin/:$PATH \
+    PYTHONUNBUFFERED=1 \
+    PYTHONIOENCODING=UTF-8 \
+    LC_ALL=en_US.UTF-8 \
+    LANG=en_US.UTF-8 \
+    PIP_NO_CACHE_DIR=off
+
+RUN python3.11 -m venv --system-site-packages ${APP_ROOT} && \
+    chown -R 1001:0 ${APP_ROOT} && \
+    fix-permissions ${APP_ROOT} -P && \
+    rpm-file-permissions && \
+    echo "unset BASH_ENV PROMPT_COMMAND ENV" >> ${APP_ROOT}/bin/activate
+
+ENV BASH_ENV="${APP_ROOT}/bin/activate" \
+    ENV="${APP_ROOT}/bin/activate" \
+    PROMPT_COMMAND=". ${APP_ROOT}/bin/activate"
+
+# Apparently the following is needed due to training options.
+# More information can be seen in the official documentation:
+#   - https://instructlab.readthedocs.io/latest/habana-gaudi.html#install-and-run-instructlab-with-intel-gaudi
+ENV TSAN_OPTIONS='ignore_noninstrumented_modules=1'
+
+WORKDIR /opt/app-root/bin
+
+# Install micropipenv to deploy packages from Pipfile.lock
+RUN pip install --no-cache-dir -U "micropipenv[toml]" && \
+chmod -R g+w /opt/app-root/lib/python3.11/site-packages && \
+fix-permissions /opt/app-root -P
+
+# Install Python dependencies from Pipfile.lock file
+COPY Pipfile.lock ./
+
+RUN echo "Installing softwares and packages" && micropipenv install && rm -f ./Pipfile.lock
+
+# OS Packages needs to be installed as root
+USER root
+
+# Install usefull OS packages
+RUN dnf install -y mesa-libGL && dnf clean all && rm -rf /var/cache/yum
+
+# Install the oc client
+RUN curl -L https://mirror.openshift.com/pub/openshift-v4/$(uname -m)/clients/ocp/stable/openshift-client-linux.tar.gz \
+-o /tmp/openshift-client-linux.tar.gz && \
+tar -xzvf /tmp/openshift-client-linux.tar.gz oc && \
+rm -f /tmp/openshift-client-linux.tar.gz
+
+# Other apps and tools installed as default user
+USER 1001
+
+WORKDIR /opt/app-root/src

base/habana-python-3.11/Pipfile

@@ -0,0 +1,14 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+# Base packages
+wheel = "~=0.43.0"
+setuptools = "~=69.2.0"
+
+[requires]
+python_version = "3.11"

base/ubi9-python-3.11/Pipfile

@@ -6,6 +6,15 @@ verify_ssl = true
 [dev-packages]
 
 [packages]
+# JupyterLab packages
+jupyterlab = "~=3.6.7" # Wait on upgrade till plugins are ready
+jupyter-server = "~=2.13.0"
+jupyter-server-proxy = "~=4.0.0" # Upgrade would bring in jupyterlab 4
+jupyter-server-terminals = "~=0.5.3"
+jupyterlab-git = "~=0.44.0"
+nbdime = "~=3.2.1"
+nbgitpuller = "~=1.2.0"
+
 # Base packages
 wheel = "~=0.43.0"
 setuptools = "~=69.2.0"

habana/1.10.0/ubi8-python-3.8/README.md

@@ -1,8 +1,7 @@
 # Habana Notebook Image
-This directory contains the Dockerfile to build a Notebook image compatible with 1.10.0 Habana Version.  
+This directory contains the Dockerfile to build a Notebook image compatible with 1.10.0 Habana Version.
 
-Currently supporting the support matrix:  
-https://docs.habana.ai/en/latest/Support_Matrix/Support_Matrix_v1.10.0.html   
+- [Habana 1.10.0 support matrix](https://docs.habana.ai/en/latest/Support_Matrix/Support_Matrix_v1.10.0.html#support-matrix-1-10-0)
 
 
 | Firmware                 | version          |
@@ -19,9 +18,8 @@ https://docs.habana.ai/en/latest/Support_Matrix/Support_Matrix_v1.10.0.html
 | numpy          | 1.23.5  |
 | scipy          | 1.10.1  |
 
-Reference:  
-https://github.com/HabanaAI/Setup_and_Install/tree/1.10.0 
+Reference:
+https://github.com/HabanaAI/Setup_and_Install/tree/1.10.0
 
-
-For further documentation related to HabanaAI, please refer:  
-https://docs.habana.ai/en/v1.10.0/Gaudi_Overview/index.html
+For further documentation related to HabanaAI, please refer:
+https://docs.habana.ai/en/v1.10.0/Gaudi_Overview/index.html

habana/1.13.0/ubi8-python-3.8/README.md

@@ -1,8 +1,7 @@
 # Habana Notebook Image
-This directory contains the Dockerfile to build a Notebook image compatible with 1.13.0 Habana Version.  
+This directory contains the Dockerfile to build a Notebook image compatible with 1.13.0 Habana Version.
 
-Currently supporting the support matrix:  
-https://docs.habana.ai/en/latest/Support_Matrix/Support_Matrix.html
+- [Habana 1.13.0 support matrix](https://docs.habana.ai/en/latest/Support_Matrix/Support_Matrix_v1.13.0.html#support-matrix-1-13-0)
 
 | Firmware                 | version          |
 | ------------------------ | ---------------- |
@@ -18,9 +17,8 @@ https://docs.habana.ai/en/latest/Support_Matrix/Support_Matrix.html
 | numpy          | 1.23.5  |
 | scipy          | 1.10.1  |
 
-Reference:  
+Reference:
 https://github.com/HabanaAI/Setup_and_Install/tree/1.13.0
 
-
-For further documentation related to HabanaAI, please refer:  
-https://docs.habana.ai/en/v1.13.0/Gaudi_Overview/index.html
+For further documentation related to HabanaAI, please refer:
+https://docs.habana.ai/en/v1.13.0/Gaudi_Overview/index.html

habana/1.17.1/ubi9-python-3.11/Dockerfile

@@ -0,0 +1,47 @@
+
+# Copyright (c) 2024 Habana Labs, Ltd.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+USER root
+
+## Label the image with details required by ODH
+LABEL name="odh-notebook-habana-jupyter-1.17.1-ubi9-python-3.11" \
+    summary="Jupyter HabanaAI 1.17.1 notebook image for ODH notebooks" \
+    description="Jupyter HabanaAI 1.17.1 notebook image with base Python 3.11 builder image based on ubi9 for ODH notebooks" \
+    io.k8s.display-name="Jupyter HabanaAI 1.17.1 notebook image for ODH notebooks" \
+    io.k8s.description="Jupyter HabanaAI 1.17.1 notebook image with base Python 3.11 builder image based on ubi9 for ODH notebooks" \
+    authoritative-source-url="https://github.com/opendatahub-io/notebooks" \
+    io.openshift.build.commit.ref="main" \
+    io.openshift.build.source-location="https://github.com/opendatahub-io/notebooks/tree/main/habana/1.17.1/ubi9-python-3.11" \
+    io.openshift.build.image="quay.io/opendatahub/workbench-images:habana-jupyter-1.17.1-ubi9-python-3.11"
+
+WORKDIR /opt/app-root/bin
+
+COPY utils utils/
+
+COPY Pipfile.lock start-notebook.sh ./
+
+# Install Python dependencies from Pipfile.lock file
+RUN echo "Installing softwares and packages" && micropipenv install && rm -f ./Pipfile.lock
+
+# Disable announcement plugin of jupyterlab
+RUN jupyter labextension disable "@jupyterlab/apputils-extension:announcements"
+
+# Fix permissions to support pip in Openshift environments
+RUN chmod -R g+w /opt/app-root/lib/python3.11/site-packages && \
+    fix-permissions /opt/app-root -P
+
+WORKDIR /opt/app-root/src
+
+# Replace Notebook's launcher, "(ipykernel)" with Python's version 3.x.y
+RUN sed -i -e "s/Python.*/$(python --version | cut -d '.' -f-2)\",/" /opt/app-root/share/jupyter/kernels/python3/kernel.json
+
+USER 1001
+
+ENTRYPOINT ["start-notebook.sh"]
+
+WORKDIR /opt/app-root/src

habana/1.17.1/ubi9-python-3.11/Pipfile

@@ -0,0 +1,23 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+# JupyterLab packages
+jupyterlab = "~=3.6.7" # Wait on upgrade till plugins are ready
+jupyter-server = "~=2.13.0"
+jupyter-server-proxy = "~=4.0.0" # Upgrade would bring in jupyterlab 4
+jupyter-server-terminals = "~=0.5.3"
+jupyterlab-git = "~=0.44.0"
+nbdime = "~=3.2.1"
+nbgitpuller = "~=1.2.0"
+
+# Base packages
+wheel = "~=0.43.0"
+setuptools = "~=69.2.0"
+
+[requires]
+python_version = "3.11"