Skip to content

Commit

Permalink
WIP: Enable publishing to pypi
Browse files Browse the repository at this point in the history
  • Loading branch information
@leavauchier
leavauchier committed Apr 23, 2024
1 parent 4667f94 commit d8f9702
Show file tree
Hide file tree
Showing 3 changed files with 167 additions and 111 deletions.
241 changes: 144 additions & 97 deletions .github/workflows/cicd.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,106 +11,153 @@ on:
branches:
- main
- staging-*
env:
http_proxy: ${{ secrets.PROXY_URL }}
https_proxy: ${{ secrets.PROXY_URL }}
HTTP_PROXY: ${{ secrets.PROXY_URL }}
HTTPS_PROXY: ${{ secrets.PROXY_URL }}

jobs:
CICD:
# CICD:
# runs-on: self-hosted

# steps:
# - name: Checkout branch
# uses: actions/checkout@v4

# - name: Build docker image
# run: docker build --build-arg http_proxy=${{ secrets.PROXY_URL }} --build-arg https_proxy=${{ secrets.PROXY_URL }} -t myria3d .

# - name: Run pytest
# run: >
# docker run
# --ipc=host
# myria3d
# python -m
# pytest -rA -v
# --ignore=actions-runner

# # IMPORTANT: Always run images with --ipc=host and --shm-size=2gb (at least) to enable
# # sufficient shared memory when predicting on large files.
# - name: Example inference run via Docker with default config and checkpoint
# run: >
# docker run
# -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/inputs/:/inputs/
# -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/outputs/:/outputs/
# --ipc=host
# --shm-size=2gb
# myria3d
# python run.py
# predict.src_las=/inputs/792000_6272000_subset_buildings.las
# datamodule.epsg=2154
# predict.output_dir=/outputs/
# task.task_name=predict

# # predict.subtile_overlap specifies overlap between adjacent samples (in meters).
# - name: Example inference run via Docker with inference-time subtiles overlap to smooth-out results.
# run: >
# docker run
# -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/inputs/:/inputs/
# -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/outputs/:/outputs/
# --ipc=host
# --shm-size=2gb
# myria3d
# python run.py
# --config-path /inputs/
# --config-name proto151_V2.0_epoch_100_Myria3DV3.1.0_predict_config_V3.7.0
# predict.ckpt_path=/inputs/proto151_V2.0_epoch_100_Myria3DV3.1.0.ckpt
# datamodule.epsg=2154
# predict.src_las=/inputs/792000_6272000_subset_buildings.las
# predict.output_dir=/outputs/
# predict.subtile_overlap=25
# datamodule.batch_size=10
# predict.interpolator.probas_to_save=[building,ground]
# task.task_name=predict

# - name: Check code neatness (linter)
# run: docker run myria3d python -m flake8

# # Everything ran so we tag the valid docker image to keep it
# # This happens for push events, which are in particular
# # triggered when a pull request is merged.
# - name: Tag the docker image with branch name
# if: github.event_name == 'push'
# run: |
# docker tag myria3d:latest myria3d:${{github.ref_name}}
# docker run myria3d:${{github.ref_name}} bash # Run the new, tagged image at least once so that is it not prunned by mistake when using docker system prune
# # docker save myria3d:${{github.ref_name}} -o /var/data/cicd/CICD_github_assets/CICD_docker_images/myria3d_${github.ref_name}.tar # Save the docker image as myria3d_${github.ref_name}.tar

# # get version number and date, to tag the image pushed to a private docker registry
# - name: get version number
# id: tag
# run: |
# echo "VERSION=$(docker run myria3d python -m myria3d._version)" >> $GITHUB_ENV
# echo "DATE=$(date '+%Y.%m.%d')" >> $GITHUB_ENV

# # show possible tags, for debugging purpose
# - name: Print tags
# run: |
# echo "${{ env.VERSION }}"
# echo "${{ env.DATE }}"

# - name: push main docker on nexus (tagged with a date)
# # we push on nexus an image from the main branch when it has been updated (push or accepted pull request)
# if: ((github.ref_name == 'main') && (github.event_name == 'push'))
# run: |
# docker tag myria3d ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{ env.DATE }}
# docker login ${{ secrets.DOCKER_REGISTRY }} --username svc_lidarhd --password ${{ secrets.PASSWORD_SVC_LIDARHD }}
# docker push ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{ env.DATE }}

# - name: push branch docker on nexus (tagged with the branch name)
# # we push on nexus an image from a branch when it's pushed
# if: ((github.event_name == 'push') && (github.ref_name != 'main'))
# run: |
# docker tag myria3d ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{github.ref_name}}
# docker login ${{ secrets.DOCKER_REGISTRY }} --username svc_lidarhd --password ${{ secrets.PASSWORD_SVC_LIDARHD }}
# docker push ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{github.ref_name}}

# - name: Clean dangling docker images
# if: always() # always do it, even if something failed
# run: docker system prune --force # remove dangling docker images, without asking user for confirmation

publish-pypi:
runs-on: self-hosted
environment:
name: pypi
url: https://pypi.org/p/myria3d/
permissions:
contents: read
packages: write
id-token: write # IMPORTANT: this permission is mandatory for trusted publishing

steps:
- name: Checkout branch
uses: actions/checkout@v4

# See https://github.com/marketplace/actions/setup-micromamba
- name: setup-micromamba
uses: mamba-org/[email protected]
with:
environment-file: environment.yml
environment-name: myria3d # activate the environment
cache-environment: true
cache-downloads: true
generate-run-shell: true

- name: Run tests with pytest
shell: micromamba-shell {0}
run: python -m pytest ./test -s --log-cli-level DEBUG

- name: Build pip package
shell: micromamba-shell {0}
run: >
rm -rf tmp
rm -rf ign_pdal_tools.egg-info
rm -rf dist
python -m build
- name: Checkout branch
uses: actions/checkout@v4

- name: Build docker image
run: docker build --build-arg http_proxy=${{ secrets.PROXY_URL }} --build-arg https_proxy=${{ secrets.PROXY_URL }} -t myria3d .

- name: Run pytest
run: >
docker run
--ipc=host
myria3d
python -m
pytest -rA -v
--ignore=actions-runner
# IMPORTANT: Always run images with --ipc=host and --shm-size=2gb (at least) to enable
# sufficient shared memory when predicting on large files.
- name: Example inference run via Docker with default config and checkpoint
run: >
docker run
-v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/inputs/:/inputs/
-v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/outputs/:/outputs/
--ipc=host
--shm-size=2gb
myria3d
python run.py
predict.src_las=/inputs/792000_6272000_subset_buildings.las
datamodule.epsg=2154
predict.output_dir=/outputs/
task.task_name=predict
# predict.subtile_overlap specifies overlap between adjacent samples (in meters).
- name: Example inference run via Docker with inference-time subtiles overlap to smooth-out results.
run: >
docker run
-v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/inputs/:/inputs/
-v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/outputs/:/outputs/
--ipc=host
--shm-size=2gb
myria3d
python run.py
--config-path /inputs/
--config-name proto151_V2.0_epoch_100_Myria3DV3.1.0_predict_config_V3.7.0
predict.ckpt_path=/inputs/proto151_V2.0_epoch_100_Myria3DV3.1.0.ckpt
datamodule.epsg=2154
predict.src_las=/inputs/792000_6272000_subset_buildings.las
predict.output_dir=/outputs/
predict.subtile_overlap=25
datamodule.batch_size=10
predict.interpolator.probas_to_save=[building,ground]
task.task_name=predict
- name: Check code neatness (linter)
run: docker run myria3d python -m flake8

# Everything ran so we tag the valid docker image to keep it
# This happens for push events, which are in particular
# triggered when a pull request is merged.
- name: Tag the docker image with branch name
if: github.event_name == 'push'
run: |
docker tag myria3d:latest myria3d:${{github.ref_name}}
docker run myria3d:${{github.ref_name}} bash # Run the new, tagged image at least once so that is it not prunned by mistake when using docker system prune
# docker save myria3d:${{github.ref_name}} -o /var/data/cicd/CICD_github_assets/CICD_docker_images/myria3d_${github.ref_name}.tar # Save the docker image as myria3d_${github.ref_name}.tar
# get version number and date, to tag the image pushed to a private docker registry
- name: get version number
id: tag
run: |
echo "VERSION=$(docker run lidar_prod python -m myria3d._version)" >> $GITHUB_ENV
echo "DATE=$(date '+%Y.%m.%d')" >> $GITHUB_ENV
# show possible tags, for debugging purpose
- name: Print tags
run: |
echo "${{ env.VERSION }}"
echo "${{ env.DATE }}"
- name: push main docker on nexus (tagged with a date)
# we push on nexus an image from the main branch when it has been updated (push or accepted pull request)
if: ((github.ref_name == 'main') && (github.event_name == 'push'))
run: |
docker tag myria3d ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{ env.DATE }}
docker login ${{ secrets.DOCKER_REGISTRY }} --username svc_lidarhd --password ${{ secrets.PASSWORD_SVC_LIDARHD }}
docker push ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{ env.DATE }}
- name: push branch docker on nexus (tagged with the branch name)
# we push on nexus an image from a branch when it's pushed
if: ((github.event_name == 'push') && (github.ref_name != 'main'))
run: |
docker tag myria3d ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{github.ref_name}}
docker login ${{ secrets.DOCKER_REGISTRY }} --username svc_lidarhd --password ${{ secrets.PASSWORD_SVC_LIDARHD }}
docker push ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{github.ref_name}}
- name: Clean dangling docker images
if: always() # always do it, even if something failed
run: docker system prune --force # remove dangling docker images, without asking user for confirmation
- name: pypi-publish
if: ((github.ref_name == 'main') && (github.event_name == 'push'))
uses: pypa/[email protected]
with:
skip-existing: true
18 changes: 13 additions & 5 deletions docs/source/tutorials/install_on_linux.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

### Prerequisites

We use [anaconda](https://www.anaconda.com/products/individual) to manage virtual environments.
We use [anaconda](https://www.anaconda.com/products/individual) to manage virtual environments.
This makes installing pytorch-related libraries way easier than using pure pip installs.

We enable CUDA-acceleration in pytorch as part of the defaut virtual environment recipe (see below).
Expand Down Expand Up @@ -43,20 +43,28 @@ Start by activating the virtual environment with
```bash
conda activate myria3d
```
Then install from a specific branch from github directly. Argument `branch_name` might be `main`, `dev`, or a specific release.

Then install the latest version from pypi.
**Warning:** activating the environment is required as the public pip package does not
handle its dependencies!
```
pip install --upgrade https://github.com/IGNF/myria3d/tarball/{branch_name}
pip install myria3d
```

Or install from a specific branch from github directly. Argument `branch_name` might be `main`, `dev`, or a specific release.
```
pip install --upgrade https://github.com/IGNF/myria3d/tarball/{branch_name}
```

Alternatively, you can install from sources directly in editable mode with
```bash
pip install --editable .
```


## Troubleshooting

- *OSError(libcusparse.so.11 cannot open shared object file no such file or directory)* ([**](https://github.com/pyg-team/pytorch_geometric/issues/2040#issuecomment-766610625))
- open the .bashrc file from your Ubuntu home directory and at the end of the file, add the following line (replace anaconda3 with miniconda3 if needed)

export LD_LIBRARY_PATH="/home/${USER}/anaconda3/envs/myria3d/lib:$LD_LIBRARY_PATH"
export LD_LIBRARY_PATH="/home/${USER}/anaconda3/envs/myria3d/lib:$LD_LIBRARY_PATH"
19 changes: 10 additions & 9 deletions environment.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ dependencies:
- matplotlib
# --------- loggers --------- #
- comet_ml::comet_ml=3.35
- conda-forge::urllib3<2 # To solve for https://github.com/GeneralMills/pytrends/issues/591
- conda-forge::urllib3<2 # To solve for https://github.com/GeneralMills/pytrends/issues/591
# --------- Visualization --------- #
- pandas
- matplotlib
Expand All @@ -57,17 +57,18 @@ dependencies:
- rich # beautiful text formatting in terminal
- sh # for running bash commands in some tests
- pudb # debugger
- twine # to publish pip package
# # --------- Documentation --------- #
- sphinx==4.5.*
- recommonmark==0.7.*
- sphinx_rtd_theme==1.0.*
- docutils==0.17
- rstcheck==3.3.* # RST Linter
- rstcheck==3.3.* # RST Linter
- pip:
- hydra-core==1.1.*
- hydra-colorlog==1.1.*
# --------- Documentation --------- #
- myst_parser==0.17.*
- sphinxnotes-mock==1.0.0b0 # still a beta
- sphinx_paramlinks==0.5.*
- ign-pdal-tools>=1.5.2
- hydra-core==1.1.*
- hydra-colorlog==1.1.*
# --------- Documentation --------- #
- myst_parser==0.17.*
- sphinxnotes-mock==1.0.0b0 # still a beta
- sphinx_paramlinks==0.5.*
- ign-pdal-tools>=1.5.2

0 comments on commit d8f9702

Please sign in to comment.