From 31e37058d778cfa34ac07c9b56a28dc55c8ec4fc Mon Sep 17 00:00:00 2001 From: Carlo Date: Thu, 15 May 2025 23:58:58 +0200 Subject: [PATCH 01/76] u --- docker/Dockerfile | 21 +++++++++++++++++++++ docker/docker-compose.yaml | 10 ++++++++++ docker/requirements.txt | 7 +++++++ 3 files changed, 38 insertions(+) create mode 100644 docker/Dockerfile create mode 100644 docker/docker-compose.yaml create mode 100644 docker/requirements.txt diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..3689048db --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,21 @@ +# --------------------------------------------------------------------------------- +# Universal multi-arch image (works on both Apple Silicon & Intel) +# --------------------------------------------------------------------------------- +FROM quay.io/jupyter/datascience-notebook:python-3.11 +# ---------- system utilities ------------------------------------------------------ +USER root +RUN apt-get update && \ + apt-get install -y --no-install-recommends git && \ + rm -rf /var/lib/apt/lists/* +USER ${NB_UID} + +# ---------- copy the course ------------------------------------------------------- +WORKDIR /home/jovyan/work +COPY .. . + +# ---------- extra Python deps ----------------------------------------------------- +COPY requirements.txt /tmp/ +RUN pip install --no-cache-dir -r /tmp/requirements.txt + +EXPOSE 8888 +CMD ["start-notebook.sh"] \ No newline at end of file diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 000000000..f932bef2f --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,10 @@ +version: "3.9" +services: + dlcc: + build: . + ports: + - "8888:8888" + volumes: + - ../:/home/jovyan/work # bind-mount whole repo ⇒ edits persist + environment: + - JUPYTER_ENABLE_LAB=yes diff --git a/docker/requirements.txt b/docker/requirements.txt new file mode 100644 index 000000000..9d7ef11e5 --- /dev/null +++ b/docker/requirements.txt @@ -0,0 +1,7 @@ +torch==2.7.* +torchvision +torchaudio +deeptrack +deeplay +pytorch-lightning +numpy From e1a80f48c7dc207c1753b4d2188074f323ad884a Mon Sep 17 00:00:00 2001 From: Carlo Date: Fri, 16 May 2025 11:52:17 +0200 Subject: [PATCH 02/76] u --- .github/workflows/docker-publish.yml | 42 ++++++++++++++++++++++++++++ docker/docker-compose.yaml | 1 - 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/docker-publish.yml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 000000000..f598de146 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,42 @@ +name: Build & Publish Docker Image + +on: + push: + branches: + - main + paths: + - 'docker/**' + - '.github/workflows/docker-publish.yml' + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read # for checkout + packages: write # to push to GHCR + steps: + - uses: actions/checkout@v3 + + - name: Set up QEMU (for multi-arch builds) + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v3 + with: + context: . # or ./docker if you keep your Dockerfile there + file: ./docker/Dockerfile # adjust path if needed + push: true + platforms: linux/amd64,linux/arm64 + tags: | + ghcr.io/${{ github.repository_owner }}/deep-learning-crash-course:latest + ghcr.io/${{ github.repository_owner }}/deep-learning-crash-course:${{ github.sha }} diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index f932bef2f..82145c046 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,4 +1,3 @@ -version: "3.9" services: dlcc: build: . From 04726cd82cffe978d16660fa9e658303095352bb Mon Sep 17 00:00:00 2001 From: Carlo Date: Fri, 16 May 2025 14:43:43 +0200 Subject: [PATCH 03/76] v0 --- .github/workflows/docker-publish.yml | 11 ++++++++++- docker/README.md | 13 +++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 docker/README.md diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index f598de146..abd1165e9 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -1,12 +1,21 @@ name: Build & Publish Docker Image on: + + pull_request: + branches: + - docker + paths: + - 'docker/**' + - '.github/workflows/docker-publish.yml' push: branches: - - main + - docker paths: - 'docker/**' - '.github/workflows/docker-publish.yml' + paths-ignore: + - 'docker/README.md' jobs: build-and-push: diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..f59441205 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,13 @@ +# Deep Learning Crash Course + +[![Early Access - Use Code PREORDER for 25% Off](https://img.shields.io/badge/Early%20Access%20Now%20Available-Use%20Code%20PREORDER%20for%2025%25%20Off-orange)](https://nostarch.com/deep-learning-crash-course) +by Benjamin Midtvedt, Jesús Pineda, Henrik Klein Moberg, Harshith Bachimanchi, Joana B. Pereira, Carlo Manzo, Giovanni Volpe +No Starch Press, San Francisco (CA), 2025 +ISBN-13: 9781718503922 +[https://nostarch.com/deep-learning-crash-course](https://nostarch.com/deep-learning-crash-course) + +--- + +![Docker Image CI](https://github.com/DeepTrackAI/DeepLearningCrashCourse/actions/workflows/docker-publish.yml/badge.svg) + +--- From ded9661c0c03bab79af6376dde8b77bcf16875fc Mon Sep 17 00:00:00 2001 From: Carlo Date: Fri, 16 May 2025 14:49:09 +0200 Subject: [PATCH 04/76] u --- .github/workflows/docker-publish.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index abd1165e9..9048bd7a4 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -1,21 +1,24 @@ name: Build & Publish Docker Image on: - + pull_request: branches: - docker paths: - - 'docker/**' + - 'docker/Dockerfile' + - 'docker/requirements.txt' + - 'docker/docker-compose.yml' - '.github/workflows/docker-publish.yml' + push: branches: - docker paths: - - 'docker/**' + - 'docker/Dockerfile' + - 'docker/requirements.txt' + - 'docker/docker-compose.yml' - '.github/workflows/docker-publish.yml' - paths-ignore: - - 'docker/README.md' jobs: build-and-push: From f10ad8dc65b69797e943d140f49725ec5330e96b Mon Sep 17 00:00:00 2001 From: Carlo Date: Fri, 16 May 2025 14:53:26 +0200 Subject: [PATCH 05/76] little fixes --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 9048bd7a4..52fbbebf6 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -50,5 +50,5 @@ jobs: push: true platforms: linux/amd64,linux/arm64 tags: | - ghcr.io/${{ github.repository_owner }}/deep-learning-crash-course:latest - ghcr.io/${{ github.repository_owner }}/deep-learning-crash-course:${{ github.sha }} + ghcr.io/deeptrackai/deep-learning-crash-course:latest + ghcr.io/deeptrackai/deep-learning-crash-course:${{ github.sha }} From e9b49c0146d361aea9c6f76675e135211b323417 Mon Sep 17 00:00:00 2001 From: Carlo Date: Fri, 16 May 2025 14:58:07 +0200 Subject: [PATCH 06/76] more fixes --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 3689048db..b8dc8a2d5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,7 +14,7 @@ WORKDIR /home/jovyan/work COPY .. . # ---------- extra Python deps ----------------------------------------------------- -COPY requirements.txt /tmp/ +COPY docker/requirements.txt /tmp/ RUN pip install --no-cache-dir -r /tmp/requirements.txt EXPOSE 8888 From bcc77124092a4b8982e45c74b2549fda372a73b5 Mon Sep 17 00:00:00 2001 From: Carlo Date: Fri, 16 May 2025 15:36:49 +0200 Subject: [PATCH 07/76] gpu wheels issue --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b8dc8a2d5..3513eb88e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # --------------------------------------------------------------------------------- # Universal multi-arch image (works on both Apple Silicon & Intel) # --------------------------------------------------------------------------------- -FROM quay.io/jupyter/datascience-notebook:python-3.11 +FROM quay.io/jupyter/pytorch-notebook:cuda-12.3.1-cudnn8-runtime-ubuntu22.04-py3.11 # ---------- system utilities ------------------------------------------------------ USER root RUN apt-get update && \ From 530088f2999266303529aa34ebebfff1f9922f99 Mon Sep 17 00:00:00 2001 From: Carlo Date: Fri, 16 May 2025 15:53:43 +0200 Subject: [PATCH 08/76] cpu only --- docker/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/requirements.txt b/docker/requirements.txt index 9d7ef11e5..71806c090 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1,4 +1,4 @@ -torch==2.7.* +torch==2.7.*+cpu torchvision torchaudio deeptrack From 5c93ad452eca851eeaafd7cb9fd417d6e61d2107 Mon Sep 17 00:00:00 2001 From: Carlo Date: Fri, 16 May 2025 15:55:39 +0200 Subject: [PATCH 09/76] cpu only --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 3513eb88e..b8dc8a2d5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # --------------------------------------------------------------------------------- # Universal multi-arch image (works on both Apple Silicon & Intel) # --------------------------------------------------------------------------------- -FROM quay.io/jupyter/pytorch-notebook:cuda-12.3.1-cudnn8-runtime-ubuntu22.04-py3.11 +FROM quay.io/jupyter/datascience-notebook:python-3.11 # ---------- system utilities ------------------------------------------------------ USER root RUN apt-get update && \ From a6005118d37c95ad84dda7ed00364c6df0886024 Mon Sep 17 00:00:00 2001 From: Carlo Date: Fri, 16 May 2025 15:59:50 +0200 Subject: [PATCH 10/76] cpu only --- docker/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/requirements.txt b/docker/requirements.txt index 71806c090..81db8e2b4 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1,4 +1,4 @@ -torch==2.7.*+cpu +torch==2.7.1+cpu torchvision torchaudio deeptrack From b75face48a8a6896737c2e878775ae6082402077 Mon Sep 17 00:00:00 2001 From: Carlo Date: Fri, 16 May 2025 16:57:04 +0200 Subject: [PATCH 11/76] cpu only --- docker/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/requirements.txt b/docker/requirements.txt index 81db8e2b4..159c4737a 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1,4 +1,4 @@ -torch==2.7.1+cpu +torch==2.7.0+cpu torchvision torchaudio deeptrack From 8fdb829fb8545b00daafb0163e69c6a40a3de270 Mon Sep 17 00:00:00 2001 From: Carlo Date: Fri, 16 May 2025 17:07:31 +0200 Subject: [PATCH 12/76] cpu only --- docker/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/requirements.txt b/docker/requirements.txt index 159c4737a..d6a856f50 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1,6 +1,6 @@ -torch==2.7.0+cpu -torchvision -torchaudio +torch==2.6.0+cpu +torchvision==0.21.0+cpu +torchaudio==2.6.0+cpu deeptrack deeplay pytorch-lightning From 6e4b33cede5cbc11365dbef419d41882f7fc43c1 Mon Sep 17 00:00:00 2001 From: Carlo Date: Fri, 16 May 2025 17:22:54 +0200 Subject: [PATCH 13/76] cpu only --- docker/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/requirements.txt b/docker/requirements.txt index d6a856f50..a56549ea5 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1,6 +1,6 @@ -torch==2.6.0+cpu -torchvision==0.21.0+cpu -torchaudio==2.6.0+cpu +torch==2.7.0 +torchvision==0.22.0 +torchaudio==2.7.0 deeptrack deeplay pytorch-lightning From 91474be80c85e9437fc4e297d858276c30a795c6 Mon Sep 17 00:00:00 2001 From: Carlo Date: Fri, 16 May 2025 17:29:16 +0200 Subject: [PATCH 14/76] cpu only --- docker/requirements.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/requirements.txt b/docker/requirements.txt index a56549ea5..7cd644e0d 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1,6 +1,7 @@ -torch==2.7.0 -torchvision==0.22.0 -torchaudio==2.7.0 +--index-url https://download.pytorch.org/whl/cpu +torch==2.6.0 +torchvision==0.21.0 +torchaudio==2.6.0 deeptrack deeplay pytorch-lightning From 08bc69cee50c15e5385c59c74991fca87dfc7239 Mon Sep 17 00:00:00 2001 From: Carlo Date: Fri, 16 May 2025 17:38:12 +0200 Subject: [PATCH 15/76] cpu only --- docker/requirements.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/requirements.txt b/docker/requirements.txt index 7cd644e0d..d5454b42a 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1,8 +1,8 @@ --index-url https://download.pytorch.org/whl/cpu -torch==2.6.0 -torchvision==0.21.0 -torchaudio==2.6.0 -deeptrack -deeplay -pytorch-lightning -numpy +torch==2.7.0 +torchvision==0.22.0 +torchaudio==2.7.0 +deeptrack==2.0.1 +deeplay==0.1.3 +pytorch-lightning==2.5.1 +numpy==2.2.5 From 800180919815a16f9f2e2cc11d01f75bd4aa472d Mon Sep 17 00:00:00 2001 From: Carlo Date: Fri, 16 May 2025 17:46:09 +0200 Subject: [PATCH 16/76] cpu only --- docker/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/requirements.txt b/docker/requirements.txt index d5454b42a..b2b2f7f05 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -2,7 +2,7 @@ torch==2.7.0 torchvision==0.22.0 torchaudio==2.7.0 -deeptrack==2.0.1 +deeptrack==2.0.0 deeplay==0.1.3 pytorch-lightning==2.5.1 numpy==2.2.5 From 08f075e010faec96a2f9fe9058e17918789a4db8 Mon Sep 17 00:00:00 2001 From: Carlo Date: Fri, 16 May 2025 18:01:24 +0200 Subject: [PATCH 17/76] cpu only --- docker/Dockerfile | 5 +++++ docker/requirements.txt | 6 +----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b8dc8a2d5..a0ba2707d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,6 +14,11 @@ WORKDIR /home/jovyan/work COPY .. . # ---------- extra Python deps ----------------------------------------------------- +# CPU wheels only +RUN pip install --no-cache-dir \ + torch==2.7.0 torchvision==0.22.0 torchaudio==2.7.0 \ + --index-url https://download.pytorch.org/whl/cpu + COPY docker/requirements.txt /tmp/ RUN pip install --no-cache-dir -r /tmp/requirements.txt diff --git a/docker/requirements.txt b/docker/requirements.txt index b2b2f7f05..913b46ff7 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1,8 +1,4 @@ ---index-url https://download.pytorch.org/whl/cpu -torch==2.7.0 -torchvision==0.22.0 -torchaudio==2.7.0 -deeptrack==2.0.0 +deeptrack==2.0.1 deeplay==0.1.3 pytorch-lightning==2.5.1 numpy==2.2.5 From 084ec47a1603414e90b1b7b985e56743b5fef14a Mon Sep 17 00:00:00 2001 From: Carlo Date: Fri, 16 May 2025 18:44:48 +0200 Subject: [PATCH 18/76] gpu only --- .github/workflows/docker-publish.yml | 49 ++++++++++++++++++---------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 52fbbebf6..3b2ae2fa8 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -3,8 +3,7 @@ name: Build & Publish Docker Image on: pull_request: - branches: - - docker + branches: [ docker ] paths: - 'docker/Dockerfile' - 'docker/requirements.txt' @@ -12,43 +11,57 @@ on: - '.github/workflows/docker-publish.yml' push: - branches: - - docker + branches: [ docker ] paths: - 'docker/Dockerfile' - 'docker/requirements.txt' - 'docker/docker-compose.yml' - '.github/workflows/docker-publish.yml' + +# ─────────────── global env ────────────────────────────────────────────────── +env: + REGISTRY: ghcr.io + IMAGE_NAME: deeptrackai/deep-learning-crash-course # change once here + jobs: build-and-push: runs-on: ubuntu-latest permissions: - contents: read # for checkout - packages: write # to push to GHCR + contents: read + packages: write # needed to push to GHCR + steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - name: Set up QEMU (for multi-arch builds) - uses: docker/setup-qemu-action@v2 + - uses: docker/setup-qemu-action@v3 # multi-arch emulation + - uses: docker/setup-buildx-action@v3 # Buildx driver with cache support - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + # -------- optional, but speeds repeat builds ----------- + - name: Restore Docker layer cache + uses: actions/cache@v4 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: ${{ runner.os }}-buildx- + # -------------------------------------------------------- - - name: Log in to GitHub Container Registry - uses: docker/login-action@v2 + - name: Log in to GHCR + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push - uses: docker/build-push-action@v3 + - name: Build (and push on main branch) + uses: docker/build-push-action@v5 with: - context: . # or ./docker if you keep your Dockerfile there - file: ./docker/Dockerfile # adjust path if needed - push: true + context: . + file: docker/Dockerfile # adjust if your Dockerfile moves platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name == 'push' }} # PRs build but don’t publish tags: | ghcr.io/deeptrackai/deep-learning-crash-course:latest ghcr.io/deeptrackai/deep-learning-crash-course:${{ github.sha }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache,mode=max \ No newline at end of file From 01f3a964091ddb52577b41d867a90716988f84c3 Mon Sep 17 00:00:00 2001 From: Carlo Date: Fri, 16 May 2025 18:52:39 +0200 Subject: [PATCH 19/76] cpu only --- .github/workflows/docker-publish.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 3b2ae2fa8..0654431f4 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -49,7 +49,7 @@ jobs: - name: Log in to GHCR uses: docker/login-action@v3 with: - registry: ghcr.io + registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} @@ -61,7 +61,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: ${{ github.event_name == 'push' }} # PRs build but don’t publish tags: | - ghcr.io/deeptrackai/deep-learning-crash-course:latest - ghcr.io/deeptrackai/deep-learning-crash-course:${{ github.sha }} + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache,mode=max \ No newline at end of file From d59dfb982569566f23d9e298e368ca0bd247806e Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 10:47:47 +0200 Subject: [PATCH 20/76] cpu only --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 0654431f4..a0419fe6d 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -61,7 +61,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: ${{ github.event_name == 'push' }} # PRs build but don’t publish tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }} + ghcr.io/deeptrackai/deep-learning-crash-course:latest + ghcr.io/deeptrackai/deep-learning-crash-course:${{ github.sha }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache,mode=max \ No newline at end of file From 8432cbcb81518650f91fef9415176afa51abe2c4 Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 11:12:43 +0200 Subject: [PATCH 21/76] cpu only --- .github/workflows/docker-publish.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a0419fe6d..d77c93629 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -33,7 +33,6 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: docker/setup-qemu-action@v3 # multi-arch emulation - uses: docker/setup-buildx-action@v3 # Buildx driver with cache support @@ -54,14 +53,15 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build (and push on main branch) - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v3 with: context: . file: docker/Dockerfile # adjust if your Dockerfile moves platforms: linux/amd64,linux/arm64 push: ${{ github.event_name == 'push' }} # PRs build but don’t publish tags: | - ghcr.io/deeptrackai/deep-learning-crash-course:latest - ghcr.io/deeptrackai/deep-learning-crash-course:${{ github.sha }} + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache,mode=max \ No newline at end of file From 67daac4eab2be25ed5e66099b6a91350af748087 Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 11:19:45 +0200 Subject: [PATCH 22/76] cpu only --- .github/workflows/docker-publish.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index d77c93629..f41189f81 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -60,7 +60,6 @@ jobs: platforms: linux/amd64,linux/arm64 push: ${{ github.event_name == 'push' }} # PRs build but don’t publish tags: | - tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }} cache-from: type=local,src=/tmp/.buildx-cache From a2c5571f3c59eafcdab5f0a9e5d8845fd4ca781a Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 11:29:40 +0200 Subject: [PATCH 23/76] cpu only --- docker/README.md | 52 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index f59441205..5d957889f 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,4 +1,4 @@ -# Deep Learning Crash Course +# Deep Learning Crash Course Docker Image [![Early Access - Use Code PREORDER for 25% Off](https://img.shields.io/badge/Early%20Access%20Now%20Available-Use%20Code%20PREORDER%20for%2025%25%20Off-orange)](https://nostarch.com/deep-learning-crash-course) by Benjamin Midtvedt, Jesús Pineda, Henrik Klein Moberg, Harshith Bachimanchi, Joana B. Pereira, Carlo Manzo, Giovanni Volpe @@ -10,4 +10,54 @@ ISBN-13: 9781718503922 ![Docker Image CI](https://github.com/DeepTrackAI/DeepLearningCrashCourse/actions/workflows/docker-publish.yml/badge.svg) +A ready-to-run JupyterLab environment with all notebooks and dependencies baked in. +Works on Intel & Apple-Silicon Macs, Linux ×86_64 & ARM64. + +--- + +## Prerequisites + +- [Docker Desktop](https://www.docker.com/products/docker-desktop) (macOS, Windows) or Docker Engine (Linux) +- (Optional) [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/) on Linux for GPU support +- (Optional) VS Code + [Dev Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) extension + + --- +## General use + +### 1. Pull the latest image + +```bash +docker pull ghcr.io/deeptrackai/deep-learning-crash-course:latest +``` +Docker will automatically select the correct architecture slice (amd64 or arm64). + +### 2. Run JupyterLab +```docker run --rm -it \ + -p 8888:8888 \ + -v "$(pwd)":/home/jovyan/work \ + ghcr.io/deeptrackai/deep-learning-crash-course:latest + ``` +After startup you’ll see a URL with a token (e.g. http://127.0.0.1:8888/lab?token=…). Paste it into your browser. + +--- + +## Use in VS Code (Dev Containers) + +### 1. Install the “Dev Containers” extension in VS Code. + +### 2. Command Palette → Dev Containers: Open Folder in Container… + +### 3. Select your local DeepLearningCrashCourse folder. + +### 4. In the new window: + + - Install Python & Jupyter extensions when prompted. + + - Open Folder → /home/jovyan/work (the repo inside the container). + + - Select Interpreter in the bottom bar → the one at `/opt/conda/bin/python` (Python 3.11). + +### 5. Use the integrated terminal (`Ctrl`+``) or Run & Debug scripts (F5). + +### 6. Open any `.ipynb` and run cells. \ No newline at end of file From df67738f59526939ff13d518da0bf31f4cf7936a Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 11:34:30 +0200 Subject: [PATCH 24/76] cpu only --- .github/workflows/docker-publish.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index f41189f81..51a96541f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -36,14 +36,14 @@ jobs: - uses: docker/setup-qemu-action@v3 # multi-arch emulation - uses: docker/setup-buildx-action@v3 # Buildx driver with cache support - # -------- optional, but speeds repeat builds ----------- - - name: Restore Docker layer cache - uses: actions/cache@v4 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: ${{ runner.os }}-buildx- - # -------------------------------------------------------- + # # -------- optional, but speeds repeat builds ----------- + # - name: Restore Docker layer cache + # uses: actions/cache@v4 + # with: + # path: /tmp/.buildx-cache + # key: ${{ runner.os }}-buildx-${{ github.sha }} + # restore-keys: ${{ runner.os }}-buildx- + # # -------------------------------------------------------- - name: Log in to GHCR uses: docker/login-action@v3 @@ -62,5 +62,5 @@ jobs: tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache,mode=max \ No newline at end of file + # cache-from: type=local,src=/tmp/.buildx-cache + # cache-to: type=local,dest=/tmp/.buildx-cache,mode=max \ No newline at end of file From 80a6822ebc1d67173e544a78b89ce1827cce327b Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 13:29:46 +0200 Subject: [PATCH 25/76] cpu only --- .github/workflows/docker-publish.yml | 12 -------- docker/Dockerfile | 7 ++++- docker/README.md | 43 ++++++++++++++++------------ 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 51a96541f..af88f1d04 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -1,7 +1,6 @@ name: Build & Publish Docker Image on: - pull_request: branches: [ docker ] paths: @@ -36,15 +35,6 @@ jobs: - uses: docker/setup-qemu-action@v3 # multi-arch emulation - uses: docker/setup-buildx-action@v3 # Buildx driver with cache support - # # -------- optional, but speeds repeat builds ----------- - # - name: Restore Docker layer cache - # uses: actions/cache@v4 - # with: - # path: /tmp/.buildx-cache - # key: ${{ runner.os }}-buildx-${{ github.sha }} - # restore-keys: ${{ runner.os }}-buildx- - # # -------------------------------------------------------- - - name: Log in to GHCR uses: docker/login-action@v3 with: @@ -62,5 +52,3 @@ jobs: tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }} - # cache-from: type=local,src=/tmp/.buildx-cache - # cache-to: type=local,dest=/tmp/.buildx-cache,mode=max \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index a0ba2707d..b7922d570 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -13,8 +13,13 @@ USER ${NB_UID} WORKDIR /home/jovyan/work COPY .. . +# ---------- ensure correct ownership --------------------------------------------- + USER root + RUN chown -R ${NB_UID}:${NB_GID} /home/jovyan/work + USER ${NB_UID} + # ---------- extra Python deps ----------------------------------------------------- -# CPU wheels only +# CPU-only PyTorch (wheels from CPU index) RUN pip install --no-cache-dir \ torch==2.7.0 torchvision==0.22.0 torchaudio==2.7.0 \ --index-url https://download.pytorch.org/whl/cpu diff --git a/docker/README.md b/docker/README.md index 5d957889f..dc708dac9 100644 --- a/docker/README.md +++ b/docker/README.md @@ -23,41 +23,48 @@ Works on Intel & Apple-Silicon Macs, Linux ×86_64 & ARM64. --- -## General use +## Quick Start -### 1. Pull the latest image +### Open Docker Desktop -```bash +### Pull the Latest Image + +``` docker pull ghcr.io/deeptrackai/deep-learning-crash-course:latest ``` Docker will automatically select the correct architecture slice (amd64 or arm64). -### 2. Run JupyterLab + +### Startup Docker Image + ```docker run --rm -it \ -p 8888:8888 \ - -v "$(pwd)":/home/jovyan/work \ - ghcr.io/deeptrackai/deep-learning-crash-course:latest - ``` -After startup you’ll see a URL with a token (e.g. http://127.0.0.1:8888/lab?token=…). Paste it into your browser. + ghcr.io/deeptrackai/deep-learning-crash-course:latest``` --- -## Use in VS Code (Dev Containers) +### Open in JupyterLab + +After startup, you’ll see a URL with a token (e.g. http://127.0.0.1:8888/lab?token=…). Paste it into your browser. You'll see the full set of Crash Course notebooks. + +--- -### 1. Install the “Dev Containers” extension in VS Code. +### Open in VS Code (Dev Containers) -### 2. Command Palette → Dev Containers: Open Folder in Container… +After startup, in VS Code (with Dev Containers installed): + + 1. Open the Command Palette. -### 3. Select your local DeepLearningCrashCourse folder. + 2. Run ```Dev Containers: Attach to Running Container...```. -### 4. In the new window: + 3. Select your container, a new VS Code window will pop up. - - Install Python & Jupyter extensions when prompted. + 4. In the new window: - - Open Folder → /home/jovyan/work (the repo inside the container). + - Install Python & Jupyter extensions when prompted. - - Select Interpreter in the bottom bar → the one at `/opt/conda/bin/python` (Python 3.11). + - Open Folder → /home/jovyan/work (the repo inside the container). -### 5. Use the integrated terminal (`Ctrl`+``) or Run & Debug scripts (F5). + - Select Interpreter in the bottom bar → the one at `/opt/conda/bin/python` (Python 3.11). -### 6. Open any `.ipynb` and run cells. \ No newline at end of file + 5. Open any `.ipynb` and run cells. \ No newline at end of file From 9bcca5c84eada3f8a66ff967fe118ad2c2fa1c19 Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 14:38:08 +0200 Subject: [PATCH 26/76] cpu only --- docker/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/README.md b/docker/README.md index dc708dac9..01cae9c59 100644 --- a/docker/README.md +++ b/docker/README.md @@ -39,7 +39,8 @@ Docker will automatically select the correct architecture slice (amd64 or arm64) ```docker run --rm -it \ -p 8888:8888 \ - ghcr.io/deeptrackai/deep-learning-crash-course:latest``` + ghcr.io/deeptrackai/deep-learning-crash-course:latest +``` --- @@ -65,6 +66,6 @@ After startup, in VS Code (with Dev Containers installed): - Open Folder → /home/jovyan/work (the repo inside the container). - - Select Interpreter in the bottom bar → the one at `/opt/conda/bin/python` (Python 3.11). + - Select Interpreter → the one at `/opt/conda/bin/python` (Python 3.11). 5. Open any `.ipynb` and run cells. \ No newline at end of file From be908d58256c6c165b57ea39872c3800619bf02a Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 15:09:52 +0200 Subject: [PATCH 27/76] cpu only --- docker/requirements.txt | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docker/requirements.txt b/docker/requirements.txt index 913b46ff7..10631db51 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -2,3 +2,24 @@ deeptrack==2.0.1 deeplay==0.1.3 pytorch-lightning==2.5.1 numpy==2.2.5 +matplotlib==3.10.3 +IPython==9.2.0 +Pillow==11.2.1 +contractions==0.1.73 +datasets==3.6.0 +huggingface_hub==0.31.4 +kornia==0.8.1 +networkx==3.4.2 +opencv-python==4.11.0.86 +pandas==2.2.3 +scipy==1.15.3 +seaborn==0.13.2 +scikit-image==0.25.2 +scikit-learn==1.6.1 +spacy==3.8.6 +tifffile==2025.5.10 +torch_geometric==2.6.1 +torchmetrics==1.6.1 +tqdm==4.67.1 +transformers==4.51.3 +warmup_scheduler==0.3 \ No newline at end of file From 69fa9c7ae4b8dbd7eb2d83a84f8bac1348968809 Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 15:27:44 +0200 Subject: [PATCH 28/76] cpu only --- docker/Dockerfile | 8 +++++++- docker/requirements.txt | 1 - 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b7922d570..e9f19ebb5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -21,9 +21,15 @@ COPY .. . # ---------- extra Python deps ----------------------------------------------------- # CPU-only PyTorch (wheels from CPU index) RUN pip install --no-cache-dir \ - torch==2.7.0 torchvision==0.22.0 torchaudio==2.7.0 \ + torch==2.7.0 torchvision==0.22.0\ --index-url https://download.pytorch.org/whl/cpu +# CPU-only PyG (and its C/C++ extensions) +RUN pip install --no-cache-dir \ + torch-geometric \ + -f https://data.pyg.org/whl/torch-2.7.0+cpu.html + + COPY docker/requirements.txt /tmp/ RUN pip install --no-cache-dir -r /tmp/requirements.txt diff --git a/docker/requirements.txt b/docker/requirements.txt index 10631db51..b38641979 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -18,7 +18,6 @@ scikit-image==0.25.2 scikit-learn==1.6.1 spacy==3.8.6 tifffile==2025.5.10 -torch_geometric==2.6.1 torchmetrics==1.6.1 tqdm==4.67.1 transformers==4.51.3 From 3162cf470c4407e507347fc5ffed51d3ab02e35d Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 16:26:33 +0200 Subject: [PATCH 29/76] cpu only --- docker/.dockerignore | 6 ++++++ docker/{docker-compose.yaml => docker-compose.yml} | 0 2 files changed, 6 insertions(+) create mode 100644 docker/.dockerignore rename docker/{docker-compose.yaml => docker-compose.yml} (100%) diff --git a/docker/.dockerignore b/docker/.dockerignore new file mode 100644 index 000000000..9662a049d --- /dev/null +++ b/docker/.dockerignore @@ -0,0 +1,6 @@ +.git +.github +.gitignore +.ipynb_checkpoints +docker-compose.yml +docker-publish.yml \ No newline at end of file diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yml similarity index 100% rename from docker/docker-compose.yaml rename to docker/docker-compose.yml From 5842a7d3378ee370c6cedaf412dbf0256db7344a Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 16:52:53 +0200 Subject: [PATCH 30/76] cpu only --- docker/.dockerignore | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 docker/.dockerignore diff --git a/docker/.dockerignore b/docker/.dockerignore deleted file mode 100644 index 9662a049d..000000000 --- a/docker/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -.git -.github -.gitignore -.ipynb_checkpoints -docker-compose.yml -docker-publish.yml \ No newline at end of file From e95cc9cc8d3224a8bd36c52d664af3cb60723d04 Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 16:55:47 +0200 Subject: [PATCH 31/76] cpu only --- .github/workflows/docker-publish.yml | 4 ++-- docker/Dockerfile | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index af88f1d04..db7f45cf8 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -45,8 +45,8 @@ jobs: - name: Build (and push on main branch) uses: docker/build-push-action@v3 with: - context: . - file: docker/Dockerfile # adjust if your Dockerfile moves + context: ./docker + file: Dockerfile # adjust if your Dockerfile moves platforms: linux/amd64,linux/arm64 push: ${{ github.event_name == 'push' }} # PRs build but don’t publish tags: | diff --git a/docker/Dockerfile b/docker/Dockerfile index e9f19ebb5..8a71deefe 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -11,7 +11,8 @@ USER ${NB_UID} # ---------- copy the course ------------------------------------------------------- WORKDIR /home/jovyan/work -COPY .. . +# COPY .. . +RUN git clone --depth 1 https://github.com/DeepTrackAI/DeepLearningCrashCourse.git . # ---------- ensure correct ownership --------------------------------------------- USER root From 48d1613261b7153c5160cae63192b320a16d12fd Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 17:15:12 +0200 Subject: [PATCH 32/76] cpu only --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index db7f45cf8..9667d94ff 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -46,7 +46,7 @@ jobs: uses: docker/build-push-action@v3 with: context: ./docker - file: Dockerfile # adjust if your Dockerfile moves + file: ./docker/Dockerfile # adjust if your Dockerfile moves platforms: linux/amd64,linux/arm64 push: ${{ github.event_name == 'push' }} # PRs build but don’t publish tags: | From f6a31dee9e486ff5b4b71e6ee702b9805fdd3813 Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 17:18:26 +0200 Subject: [PATCH 33/76] cpu only --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8a71deefe..18089649f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -31,7 +31,7 @@ RUN pip install --no-cache-dir \ -f https://data.pyg.org/whl/torch-2.7.0+cpu.html -COPY docker/requirements.txt /tmp/ +COPY ./requirements.txt /tmp/ RUN pip install --no-cache-dir -r /tmp/requirements.txt EXPOSE 8888 From 37ff5de6d49ef356aba7abb4d2edf7bdab9f606d Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 18:05:04 +0200 Subject: [PATCH 34/76] cpu and gpu --- .github/workflows/docker-publish.yml | 105 ++++++++++++++++++++------- docker/Dockerfile.gpu | 32 ++++++++ 2 files changed, 112 insertions(+), 25 deletions(-) create mode 100644 docker/Dockerfile.gpu diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 9667d94ff..910561166 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -1,54 +1,109 @@ -name: Build & Publish Docker Image +name: Build & Publish DLCC Images on: - pull_request: - branches: [ docker ] + push: + branches: [ main ] paths: - - 'docker/Dockerfile' - - 'docker/requirements.txt' - - 'docker/docker-compose.yml' + - 'docker/**' - '.github/workflows/docker-publish.yml' - - push: - branches: [ docker ] + pull_request: + branches: [ main ] paths: - - 'docker/Dockerfile' - - 'docker/requirements.txt' - - 'docker/docker-compose.yml' + - 'docker/**' - '.github/workflows/docker-publish.yml' - -# ─────────────── global env ────────────────────────────────────────────────── env: REGISTRY: ghcr.io - IMAGE_NAME: deeptrackai/deep-learning-crash-course # change once here + IMAGE_NAME: deep-learning-crash-course jobs: build-and-push: runs-on: ubuntu-latest permissions: contents: read - packages: write # needed to push to GHCR + packages: write + + strategy: + matrix: + variant: [cpu, gpu] steps: - uses: actions/checkout@v4 - - uses: docker/setup-qemu-action@v3 # multi-arch emulation - - uses: docker/setup-buildx-action@v3 # Buildx driver with cache support - - name: Log in to GHCR + - name: Setup QEMU & Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GHCR uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build (and push on main branch) + - name: Build & push ${{ matrix.variant }} image uses: docker/build-push-action@v3 with: context: ./docker - file: ./docker/Dockerfile # adjust if your Dockerfile moves - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name == 'push' }} # PRs build but don’t publish + file: ./docker/Dockerfile${{ matrix.variant == 'gpu' && '.gpu' || '' }} + platforms: linux/amd64${{ matrix.variant == 'cpu' && ',linux/arm64' || '' }} + push: ${{ github.event_name == 'push' }} tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }} + ${{ env.REGISTRY }}/deeptrackai/${{ env.IMAGE_NAME }}${{ matrix.variant == 'gpu' && '-gpu' || '' }}:latest + ${{ env.REGISTRY }}/deeptrackai/${{ env.IMAGE_NAME }}${{ matrix.variant == 'gpu' && '-gpu' || '' }}:${{ github.sha }} + + + +# name: Build & Publish Docker Image + +# on: +# pull_request: +# branches: [ docker ] +# paths: +# - 'docker/Dockerfile' +# - 'docker/requirements.txt' +# - 'docker/docker-compose.yml' +# - '.github/workflows/docker-publish.yml' + +# push: +# branches: [ docker ] +# paths: +# - 'docker/Dockerfile' +# - 'docker/requirements.txt' +# - 'docker/docker-compose.yml' +# - '.github/workflows/docker-publish.yml' + + +# # ─────────────── global env ────────────────────────────────────────────────── +# env: +# REGISTRY: ghcr.io +# IMAGE_NAME: deeptrackai/deep-learning-crash-course # change once here + +# jobs: +# build-and-push: +# runs-on: ubuntu-latest +# permissions: +# contents: read +# packages: write # needed to push to GHCR + +# steps: +# - uses: actions/checkout@v4 +# - uses: docker/setup-qemu-action@v3 # multi-arch emulation +# - uses: docker/setup-buildx-action@v3 # Buildx driver with cache support + +# - name: Log in to GHCR +# uses: docker/login-action@v3 +# with: +# registry: ${{ env.REGISTRY }} +# username: ${{ github.actor }} +# password: ${{ secrets.GITHUB_TOKEN }} + +# - name: Build (and push on main branch) +# uses: docker/build-push-action@v3 +# with: +# context: ./docker +# file: ./docker/Dockerfile # adjust if your Dockerfile moves +# platforms: linux/amd64,linux/arm64 +# push: ${{ github.event_name == 'push' }} # PRs build but don’t publish +# tags: | +# ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest +# ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }} diff --git a/docker/Dockerfile.gpu b/docker/Dockerfile.gpu new file mode 100644 index 000000000..6af5fdcc8 --- /dev/null +++ b/docker/Dockerfile.gpu @@ -0,0 +1,32 @@ +# docker/Dockerfile.gpu +# ──────────────────────────────────────────────────────────────────────────────── +# GPU-enabled Jupyter + PyTorch + your course +# ──────────────────────────────────────────────────────────────────────────────── +FROM quay.io/jupyter/pytorch-notebook:cuda12-python-3.11.9 + +USER root +RUN apt-get update && \ + apt-get install -y --no-install-recommends git && \ + rm -rf /var/lib/apt/lists/* +USER ${NB_UID} + +WORKDIR /home/jovyan/work +RUN git clone --depth 1 https://github.com/DeepTrackAI/DeepLearningCrashCourse.git . + +# ensure jovyan owns the workspace +USER root +RUN chown -R ${NB_UID}:${NB_GID} /home/jovyan/work +USER ${NB_UID} + +# Base image already has CUDA-enabled PyTorch 2.7+ +# Install PyG CUDA wheels matching torch 2.7.0+cu121 +RUN pip install --no-cache-dir \ + torch-geometric \ + -f https://data.pyg.org/whl/torch-2.7.0+cu121.html + +# Then install your extra deps +COPY ./requirements.txt /tmp/ +RUN pip install --no-cache-dir -r /tmp/requirements.txt + +EXPOSE 8888 +CMD ["start-notebook.sh"] From 79b1db889e0d623c63d4a522d6bba51519d5caea Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 18:06:34 +0200 Subject: [PATCH 35/76] cpu and gpu --- docker/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/README.md b/docker/README.md index 01cae9c59..a96fecdef 100644 --- a/docker/README.md +++ b/docker/README.md @@ -64,8 +64,8 @@ After startup, in VS Code (with Dev Containers installed): - Install Python & Jupyter extensions when prompted. - - Open Folder → /home/jovyan/work (the repo inside the container). + - Open Folder → `/home/jovyan/work` (the repo inside the container). - - Select Interpreter → the one at `/opt/conda/bin/python` (Python 3.11). + - Select Kernel → `Python Environments...` → pick the one at `/opt/conda/bin/python` (Python 3.11). 5. Open any `.ipynb` and run cells. \ No newline at end of file From a6ca0045eb8e378590a0329723875c9183ed8858 Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 18:08:06 +0200 Subject: [PATCH 36/76] cpu and gpu --- .github/workflows/docker-publish.yml | 56 ---------------------------- 1 file changed, 56 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 910561166..d225209e8 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -51,59 +51,3 @@ jobs: ${{ env.REGISTRY }}/deeptrackai/${{ env.IMAGE_NAME }}${{ matrix.variant == 'gpu' && '-gpu' || '' }}:latest ${{ env.REGISTRY }}/deeptrackai/${{ env.IMAGE_NAME }}${{ matrix.variant == 'gpu' && '-gpu' || '' }}:${{ github.sha }} - - -# name: Build & Publish Docker Image - -# on: -# pull_request: -# branches: [ docker ] -# paths: -# - 'docker/Dockerfile' -# - 'docker/requirements.txt' -# - 'docker/docker-compose.yml' -# - '.github/workflows/docker-publish.yml' - -# push: -# branches: [ docker ] -# paths: -# - 'docker/Dockerfile' -# - 'docker/requirements.txt' -# - 'docker/docker-compose.yml' -# - '.github/workflows/docker-publish.yml' - - -# # ─────────────── global env ────────────────────────────────────────────────── -# env: -# REGISTRY: ghcr.io -# IMAGE_NAME: deeptrackai/deep-learning-crash-course # change once here - -# jobs: -# build-and-push: -# runs-on: ubuntu-latest -# permissions: -# contents: read -# packages: write # needed to push to GHCR - -# steps: -# - uses: actions/checkout@v4 -# - uses: docker/setup-qemu-action@v3 # multi-arch emulation -# - uses: docker/setup-buildx-action@v3 # Buildx driver with cache support - -# - name: Log in to GHCR -# uses: docker/login-action@v3 -# with: -# registry: ${{ env.REGISTRY }} -# username: ${{ github.actor }} -# password: ${{ secrets.GITHUB_TOKEN }} - -# - name: Build (and push on main branch) -# uses: docker/build-push-action@v3 -# with: -# context: ./docker -# file: ./docker/Dockerfile # adjust if your Dockerfile moves -# platforms: linux/amd64,linux/arm64 -# push: ${{ github.event_name == 'push' }} # PRs build but don’t publish -# tags: | -# ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest -# ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }} From b15f15043aefca11c15594ae4acb436118e0b1cd Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 18:10:44 +0200 Subject: [PATCH 37/76] cpu and gpu --- .github/workflows/docker-publish.yml | 68 ++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index d225209e8..ba2c3925e 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -2,14 +2,20 @@ name: Build & Publish DLCC Images on: push: - branches: [ main ] + branches: [ docker ] paths: - - 'docker/**' + - 'docker/Dockerfile' + - 'docker/Dockerfile.gpu'' + - 'docker/requirements.txt' + - 'docker/docker-compose.yml' - '.github/workflows/docker-publish.yml' pull_request: - branches: [ main ] + branches: [ docker ] paths: - - 'docker/**' + - 'docker/Dockerfile' + - 'docker/Dockerfile.gpu'' + - 'docker/requirements.txt' + - 'docker/docker-compose.yml' - '.github/workflows/docker-publish.yml' env: @@ -51,3 +57,57 @@ jobs: ${{ env.REGISTRY }}/deeptrackai/${{ env.IMAGE_NAME }}${{ matrix.variant == 'gpu' && '-gpu' || '' }}:latest ${{ env.REGISTRY }}/deeptrackai/${{ env.IMAGE_NAME }}${{ matrix.variant == 'gpu' && '-gpu' || '' }}:${{ github.sha }} +# name: Build & Publish Docker Image + +# on: +# pull_request: +# branches: [ docker ] +# paths: +# - 'docker/Dockerfile' +# - 'docker/requirements.txt' +# - 'docker/docker-compose.yml' +# - '.github/workflows/docker-publish.yml' + +# push: +# branches: [ docker ] +# paths: +# - 'docker/Dockerfile' +# - 'docker/requirements.txt' +# - 'docker/docker-compose.yml' +# - '.github/workflows/docker-publish.yml' + + +# # ─────────────── global env ────────────────────────────────────────────────── +# env: +# REGISTRY: ghcr.io +# IMAGE_NAME: deeptrackai/deep-learning-crash-course # change once here + +# jobs: +# build-and-push: +# runs-on: ubuntu-latest +# permissions: +# contents: read +# packages: write # needed to push to GHCR + +# steps: +# - uses: actions/checkout@v4 +# - uses: docker/setup-qemu-action@v3 # multi-arch emulation +# - uses: docker/setup-buildx-action@v3 # Buildx driver with cache support + +# - name: Log in to GHCR +# uses: docker/login-action@v3 +# with: +# registry: ${{ env.REGISTRY }} +# username: ${{ github.actor }} +# password: ${{ secrets.GITHUB_TOKEN }} + +# - name: Build (and push on main branch) +# uses: docker/build-push-action@v3 +# with: +# context: ./docker +# file: ./docker/Dockerfile # adjust if your Dockerfile moves +# platforms: linux/amd64,linux/arm64 +# push: ${{ github.event_name == 'push' }} # PRs build but don’t publish +# tags: | +# ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest +# ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }} \ No newline at end of file From a80a1938aee39ea8fd254291aacc792e048f9456 Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 18:11:22 +0200 Subject: [PATCH 38/76] cpu and gpu --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index ba2c3925e..974a080fa 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -5,7 +5,7 @@ on: branches: [ docker ] paths: - 'docker/Dockerfile' - - 'docker/Dockerfile.gpu'' + - 'docker/Dockerfile.gpu' - 'docker/requirements.txt' - 'docker/docker-compose.yml' - '.github/workflows/docker-publish.yml' @@ -13,7 +13,7 @@ on: branches: [ docker ] paths: - 'docker/Dockerfile' - - 'docker/Dockerfile.gpu'' + - 'docker/Dockerfile.gpu' - 'docker/requirements.txt' - 'docker/docker-compose.yml' - '.github/workflows/docker-publish.yml' From 280c56b1e2dd393d07063513569ff8c42168e6a9 Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 18:31:06 +0200 Subject: [PATCH 39/76] cpu and gpu --- .github/workflows/docker-publish.yml | 57 +--------------------------- 1 file changed, 1 insertion(+), 56 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 974a080fa..ed1f66ad0 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -55,59 +55,4 @@ jobs: push: ${{ github.event_name == 'push' }} tags: | ${{ env.REGISTRY }}/deeptrackai/${{ env.IMAGE_NAME }}${{ matrix.variant == 'gpu' && '-gpu' || '' }}:latest - ${{ env.REGISTRY }}/deeptrackai/${{ env.IMAGE_NAME }}${{ matrix.variant == 'gpu' && '-gpu' || '' }}:${{ github.sha }} - -# name: Build & Publish Docker Image - -# on: -# pull_request: -# branches: [ docker ] -# paths: -# - 'docker/Dockerfile' -# - 'docker/requirements.txt' -# - 'docker/docker-compose.yml' -# - '.github/workflows/docker-publish.yml' - -# push: -# branches: [ docker ] -# paths: -# - 'docker/Dockerfile' -# - 'docker/requirements.txt' -# - 'docker/docker-compose.yml' -# - '.github/workflows/docker-publish.yml' - - -# # ─────────────── global env ────────────────────────────────────────────────── -# env: -# REGISTRY: ghcr.io -# IMAGE_NAME: deeptrackai/deep-learning-crash-course # change once here - -# jobs: -# build-and-push: -# runs-on: ubuntu-latest -# permissions: -# contents: read -# packages: write # needed to push to GHCR - -# steps: -# - uses: actions/checkout@v4 -# - uses: docker/setup-qemu-action@v3 # multi-arch emulation -# - uses: docker/setup-buildx-action@v3 # Buildx driver with cache support - -# - name: Log in to GHCR -# uses: docker/login-action@v3 -# with: -# registry: ${{ env.REGISTRY }} -# username: ${{ github.actor }} -# password: ${{ secrets.GITHUB_TOKEN }} - -# - name: Build (and push on main branch) -# uses: docker/build-push-action@v3 -# with: -# context: ./docker -# file: ./docker/Dockerfile # adjust if your Dockerfile moves -# platforms: linux/amd64,linux/arm64 -# push: ${{ github.event_name == 'push' }} # PRs build but don’t publish -# tags: | -# ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest -# ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }} \ No newline at end of file + ${{ env.REGISTRY }}/deeptrackai/${{ env.IMAGE_NAME }}${{ matrix.variant == 'gpu' && '-gpu' || '' }}:${{ github.sha }} \ No newline at end of file From 1902fb87d560bfd16b45a6e85e8a09ed449074b3 Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 18:32:04 +0200 Subject: [PATCH 40/76] cpu and gpu --- docker/README.md | 62 ++++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/docker/README.md b/docker/README.md index a96fecdef..f5f65be08 100644 --- a/docker/README.md +++ b/docker/README.md @@ -11,61 +11,61 @@ ISBN-13: 9781718503922 ![Docker Image CI](https://github.com/DeepTrackAI/DeepLearningCrashCourse/actions/workflows/docker-publish.yml/badge.svg) A ready-to-run JupyterLab environment with all notebooks and dependencies baked in. -Works on Intel & Apple-Silicon Macs, Linux ×86_64 & ARM64. +Works on Intel & Apple-Silicon Macs, Linux ×86_64 & ARM64; also provides an NVIDIA-CUDA-enabled variant for GPU hosts. --- ## Prerequisites -- [Docker Desktop](https://www.docker.com/products/docker-desktop) (macOS, Windows) or Docker Engine (Linux) -- (Optional) [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/) on Linux for GPU support -- (Optional) VS Code + [Dev Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) extension - +- **Docker** + - macOS / Windows → [Docker Desktop](https://www.docker.com/products/docker-desktop) + - Linux → Docker Engine + (optional) [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/) +- (Optional) **VS Code** + [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) --- -## Quick Start -### Open Docker Desktop +## Quick Start -### Pull the Latest Image +### 1. Pull the image -``` +**CPU-only (multi-arch)** +```bash docker pull ghcr.io/deeptrackai/deep-learning-crash-course:latest ``` -Docker will automatically select the correct architecture slice (amd64 or arm64). +**GPU-enabled (amd64 + CUDA)** +```bash +docker pull ghcr.io/deeptrackai/deep-learning-crash-course-gpu:latest +``` -### Startup Docker Image +### 2. Start JupyterLab -```docker run --rm -it \ +**CPU-only (multi-arch)** +```bash +docker run --rm -it \ -p 8888:8888 \ ghcr.io/deeptrackai/deep-learning-crash-course:latest ``` ---- - -### Open in JupyterLab - -After startup, you’ll see a URL with a token (e.g. http://127.0.0.1:8888/lab?token=…). Paste it into your browser. You'll see the full set of Crash Course notebooks. +**GPU-enabled (amd64 + CUDA)** +```bash +docker run --rm -it --gpus all \ + -p 8888:8888 \ + ghcr.io/deeptrackai/deep-learning-crash-course-gpu:latest +``` +After startup, copy the URL with token (e.g., http://127.0.0.1:8888/lab?token=…) into your browser to access JupyterLab. ---- -### Open in VS Code (Dev Containers) +### Attach in VS Code (Dev Containers) -After startup, in VS Code (with Dev Containers installed): + 1. In VS Code, open Command Palette (`Ctrl+Shift+P`). - 1. Open the Command Palette. - - 2. Run ```Dev Containers: Attach to Running Container...```. - - 3. Select your container, a new VS Code window will pop up. - - 4. In the new window: + 2. Run **Dev Containers: Attach to Running Container...** - - Install Python & Jupyter extensions when prompted. + 3. Select your **CPU** or **GPU** container from the list. A new VS Code window will pop up. - - Open Folder → `/home/jovyan/work` (the repo inside the container). + 4. Install Python & Jupyter extensions when prompted. - - Select Kernel → `Python Environments...` → pick the one at `/opt/conda/bin/python` (Python 3.11). + 5. **Open Folder** → `/home/jovyan/work` and **Select Kernel** → `/opt/conda/bin/python` (Python 3.11). - 5. Open any `.ipynb` and run cells. \ No newline at end of file + 6. Open any `.ipynb` and run cells. \ No newline at end of file From 12aa7b2e95ff599c71f4205cd3b58bcffd71f3e1 Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 18:34:30 +0200 Subject: [PATCH 41/76] cpu and gpu --- docker/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/README.md b/docker/README.md index f5f65be08..930516a5c 100644 --- a/docker/README.md +++ b/docker/README.md @@ -53,10 +53,10 @@ docker run --rm -it --gpus all \ -p 8888:8888 \ ghcr.io/deeptrackai/deep-learning-crash-course-gpu:latest ``` +### 3. Run in JupyterLab... After startup, copy the URL with token (e.g., http://127.0.0.1:8888/lab?token=…) into your browser to access JupyterLab. - -### Attach in VS Code (Dev Containers) +### ...or attach in VS Code (Dev Containers) 1. In VS Code, open Command Palette (`Ctrl+Shift+P`). From 45b546a6813983047307dc7d7f1f7e2670f8fb9b Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 18:38:01 +0200 Subject: [PATCH 42/76] cpu and gpu --- docker/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/README.md b/docker/README.md index 930516a5c..4ecdd142c 100644 --- a/docker/README.md +++ b/docker/README.md @@ -53,10 +53,11 @@ docker run --rm -it --gpus all \ -p 8888:8888 \ ghcr.io/deeptrackai/deep-learning-crash-course-gpu:latest ``` -### 3. Run in JupyterLab... +### 3. Run in JupyterLab ... + After startup, copy the URL with token (e.g., http://127.0.0.1:8888/lab?token=…) into your browser to access JupyterLab. -### ...or attach in VS Code (Dev Containers) +### ... or attach in VS Code (Dev Containers) 1. In VS Code, open Command Palette (`Ctrl+Shift+P`). From 9df3d7ff4c55ccff5fdb8346ed0764858ab28206 Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 18:39:02 +0200 Subject: [PATCH 43/76] cpu and gpu --- docker/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index 4ecdd142c..bcec0c849 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,4 +1,4 @@ -# Deep Learning Crash Course Docker Image +# Deep Learning Crash Course [![Early Access - Use Code PREORDER for 25% Off](https://img.shields.io/badge/Early%20Access%20Now%20Available-Use%20Code%20PREORDER%20for%2025%25%20Off-orange)](https://nostarch.com/deep-learning-crash-course) by Benjamin Midtvedt, Jesús Pineda, Henrik Klein Moberg, Harshith Bachimanchi, Joana B. Pereira, Carlo Manzo, Giovanni Volpe @@ -8,6 +8,8 @@ ISBN-13: 9781718503922 --- +# Deep Learning Crash Course Docker Image + ![Docker Image CI](https://github.com/DeepTrackAI/DeepLearningCrashCourse/actions/workflows/docker-publish.yml/badge.svg) A ready-to-run JupyterLab environment with all notebooks and dependencies baked in. From c468373db4e5e7e18bf7dd28abfedf5f673e2934 Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 18:57:44 +0200 Subject: [PATCH 44/76] cpu and gpu --- docker/Dockerfile | 9 ++++++++- docker/Dockerfile.gpu | 10 ++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 18089649f..a114ff7fa 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -30,9 +30,16 @@ RUN pip install --no-cache-dir \ torch-geometric \ -f https://data.pyg.org/whl/torch-2.7.0+cpu.html - COPY ./requirements.txt /tmp/ RUN pip install --no-cache-dir -r /tmp/requirements.txt +# Enable Jupyter widget support +RUN pip install --no-cache-dir ipywidgets + +# For JupyterLab (inside the container), install the widget manager +RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager \ + --no-build && \ + jupyter lab build + EXPOSE 8888 CMD ["start-notebook.sh"] \ No newline at end of file diff --git a/docker/Dockerfile.gpu b/docker/Dockerfile.gpu index 6af5fdcc8..3b07c6c51 100644 --- a/docker/Dockerfile.gpu +++ b/docker/Dockerfile.gpu @@ -28,5 +28,15 @@ RUN pip install --no-cache-dir \ COPY ./requirements.txt /tmp/ RUN pip install --no-cache-dir -r /tmp/requirements.txt +# Enable Jupyter widget support +RUN pip install --no-cache-dir ipywidgets + +# For JupyterLab (inside the container), install the widget manager +RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager \ + --no-build && \ + jupyter lab build && \ + jupyter labextension enable @jupyter-widgets/jupyterlab-manager && \ + jupyter labextension install @jupyter-widgets/jupyterlab-manager --no-build + EXPOSE 8888 CMD ["start-notebook.sh"] From 32be36fbcf84e056f09bd5333b2d01e2daaf3515 Mon Sep 17 00:00:00 2001 From: Carlo Date: Mon, 19 May 2025 23:54:49 +0200 Subject: [PATCH 45/76] cpu and gpu --- docker/Dockerfile | 1 - docker/Dockerfile.gpu | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index a114ff7fa..d023cd2b8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -11,7 +11,6 @@ USER ${NB_UID} # ---------- copy the course ------------------------------------------------------- WORKDIR /home/jovyan/work -# COPY .. . RUN git clone --depth 1 https://github.com/DeepTrackAI/DeepLearningCrashCourse.git . # ---------- ensure correct ownership --------------------------------------------- diff --git a/docker/Dockerfile.gpu b/docker/Dockerfile.gpu index 3b07c6c51..bffe5e9b3 100644 --- a/docker/Dockerfile.gpu +++ b/docker/Dockerfile.gpu @@ -1,4 +1,3 @@ -# docker/Dockerfile.gpu # ──────────────────────────────────────────────────────────────────────────────── # GPU-enabled Jupyter + PyTorch + your course # ──────────────────────────────────────────────────────────────────────────────── @@ -34,9 +33,7 @@ RUN pip install --no-cache-dir ipywidgets # For JupyterLab (inside the container), install the widget manager RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager \ --no-build && \ - jupyter lab build && \ - jupyter labextension enable @jupyter-widgets/jupyterlab-manager && \ - jupyter labextension install @jupyter-widgets/jupyterlab-manager --no-build + jupyter lab build EXPOSE 8888 CMD ["start-notebook.sh"] From c17827e251ced20df41d787caf9148ceb0935f02 Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 00:17:57 +0200 Subject: [PATCH 46/76] cpu and gpu --- docker/Dockerfile | 13 +++++++------ docker/Dockerfile.gpu | 14 ++++++++------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d023cd2b8..64433a052 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -32,13 +32,14 @@ RUN pip install --no-cache-dir \ COPY ./requirements.txt /tmp/ RUN pip install --no-cache-dir -r /tmp/requirements.txt -# Enable Jupyter widget support -RUN pip install --no-cache-dir ipywidgets +# Install widget support +RUN pip install --no-cache-dir \ + ipywidgets \ + jupyterlab_widgets -# For JupyterLab (inside the container), install the widget manager -RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager \ - --no-build && \ - jupyter lab build +# Classic Notebook support (optional) +RUN jupyter nbextension install --py widgetsnbextension --sys-prefix && \ + jupyter nbextension enable --py widgetsnbextension --sys-prefix EXPOSE 8888 CMD ["start-notebook.sh"] \ No newline at end of file diff --git a/docker/Dockerfile.gpu b/docker/Dockerfile.gpu index bffe5e9b3..e424dacb3 100644 --- a/docker/Dockerfile.gpu +++ b/docker/Dockerfile.gpu @@ -27,13 +27,15 @@ RUN pip install --no-cache-dir \ COPY ./requirements.txt /tmp/ RUN pip install --no-cache-dir -r /tmp/requirements.txt -# Enable Jupyter widget support -RUN pip install --no-cache-dir ipywidgets +# Install widget support +RUN pip install --no-cache-dir \ + ipywidgets \ + jupyterlab_widgets + +# Classic Notebook support (optional) +RUN jupyter nbextension install --py widgetsnbextension --sys-prefix && \ + jupyter nbextension enable --py widgetsnbextension --sys-prefix -# For JupyterLab (inside the container), install the widget manager -RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager \ - --no-build && \ - jupyter lab build EXPOSE 8888 CMD ["start-notebook.sh"] From 625d6caf63e1fcac8833b9d7129c39775e92eb04 Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 00:26:19 +0200 Subject: [PATCH 47/76] cpu and gpu --- docker/Dockerfile | 7 ++++--- docker/Dockerfile.gpu | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 64433a052..207eed01d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,9 +14,9 @@ WORKDIR /home/jovyan/work RUN git clone --depth 1 https://github.com/DeepTrackAI/DeepLearningCrashCourse.git . # ---------- ensure correct ownership --------------------------------------------- - USER root - RUN chown -R ${NB_UID}:${NB_GID} /home/jovyan/work - USER ${NB_UID} +USER root +RUN chown -R ${NB_UID}:${NB_GID} /home/jovyan/work +USER ${NB_UID} # ---------- extra Python deps ----------------------------------------------------- # CPU-only PyTorch (wheels from CPU index) @@ -29,6 +29,7 @@ RUN pip install --no-cache-dir \ torch-geometric \ -f https://data.pyg.org/whl/torch-2.7.0+cpu.html +# Then install your extra deps COPY ./requirements.txt /tmp/ RUN pip install --no-cache-dir -r /tmp/requirements.txt diff --git a/docker/Dockerfile.gpu b/docker/Dockerfile.gpu index e424dacb3..7396d3038 100644 --- a/docker/Dockerfile.gpu +++ b/docker/Dockerfile.gpu @@ -2,21 +2,23 @@ # GPU-enabled Jupyter + PyTorch + your course # ──────────────────────────────────────────────────────────────────────────────── FROM quay.io/jupyter/pytorch-notebook:cuda12-python-3.11.9 - +# ---------- system utilities ------------------------------------------------------ USER root RUN apt-get update && \ apt-get install -y --no-install-recommends git && \ rm -rf /var/lib/apt/lists/* USER ${NB_UID} +# ---------- copy the course ------------------------------------------------------- WORKDIR /home/jovyan/work RUN git clone --depth 1 https://github.com/DeepTrackAI/DeepLearningCrashCourse.git . -# ensure jovyan owns the workspace +# ---------- ensure correct ownership --------------------------------------------- USER root RUN chown -R ${NB_UID}:${NB_GID} /home/jovyan/work USER ${NB_UID} +# ---------- extra Python deps ----------------------------------------------------- # Base image already has CUDA-enabled PyTorch 2.7+ # Install PyG CUDA wheels matching torch 2.7.0+cu121 RUN pip install --no-cache-dir \ @@ -36,6 +38,5 @@ RUN pip install --no-cache-dir \ RUN jupyter nbextension install --py widgetsnbextension --sys-prefix && \ jupyter nbextension enable --py widgetsnbextension --sys-prefix - EXPOSE 8888 CMD ["start-notebook.sh"] From be57813f583c08c66641d0b9e2352ef48c127bbe Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 00:31:57 +0200 Subject: [PATCH 48/76] cpu and gpu --- docker/Dockerfile | 4 ---- docker/Dockerfile.gpu | 4 ---- 2 files changed, 8 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 207eed01d..d06457063 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -38,9 +38,5 @@ RUN pip install --no-cache-dir \ ipywidgets \ jupyterlab_widgets -# Classic Notebook support (optional) -RUN jupyter nbextension install --py widgetsnbextension --sys-prefix && \ - jupyter nbextension enable --py widgetsnbextension --sys-prefix - EXPOSE 8888 CMD ["start-notebook.sh"] \ No newline at end of file diff --git a/docker/Dockerfile.gpu b/docker/Dockerfile.gpu index 7396d3038..9c45a39be 100644 --- a/docker/Dockerfile.gpu +++ b/docker/Dockerfile.gpu @@ -34,9 +34,5 @@ RUN pip install --no-cache-dir \ ipywidgets \ jupyterlab_widgets -# Classic Notebook support (optional) -RUN jupyter nbextension install --py widgetsnbextension --sys-prefix && \ - jupyter nbextension enable --py widgetsnbextension --sys-prefix - EXPOSE 8888 CMD ["start-notebook.sh"] From fc55edd4b590b75cbcb2a0fb90e6fdbc425882d0 Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 01:22:03 +0200 Subject: [PATCH 49/76] cpu and gpu --- docker/Dockerfile | 12 ++++++++++-- docker/Dockerfile.gpu | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d06457063..fc2a5cab1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -33,10 +33,18 @@ RUN pip install --no-cache-dir \ COPY ./requirements.txt /tmp/ RUN pip install --no-cache-dir -r /tmp/requirements.txt -# Install widget support +# Install all widget bits via pip RUN pip install --no-cache-dir \ ipywidgets \ - jupyterlab_widgets + jupyterlab_widgets \ + widgetsnbextension + +# Enable classic-notebook widget nbextension +RUN jupyter nbextension install --py widgetsnbextension --sys-prefix && \ + jupyter nbextension enable --py widgetsnbextension --sys-prefix + +# Enable the server extension for JupyterLab & VS Code +RUN jupyter server extension enable --py jupyterlab_widgets --sys-prefix EXPOSE 8888 CMD ["start-notebook.sh"] \ No newline at end of file diff --git a/docker/Dockerfile.gpu b/docker/Dockerfile.gpu index 9c45a39be..493da8212 100644 --- a/docker/Dockerfile.gpu +++ b/docker/Dockerfile.gpu @@ -29,10 +29,18 @@ RUN pip install --no-cache-dir \ COPY ./requirements.txt /tmp/ RUN pip install --no-cache-dir -r /tmp/requirements.txt -# Install widget support +# Install all widget bits via pip RUN pip install --no-cache-dir \ ipywidgets \ - jupyterlab_widgets + jupyterlab_widgets \ + widgetsnbextension + +# Enable classic-notebook widget nbextension +RUN jupyter nbextension install --py widgetsnbextension --sys-prefix && \ + jupyter nbextension enable --py widgetsnbextension --sys-prefix + +# Enable the server extension for JupyterLab & VS Code +RUN jupyter server extension enable --py jupyterlab_widgets --sys-prefix EXPOSE 8888 CMD ["start-notebook.sh"] From ac73e40cb880a7a83dcbf988c55cbde471ee637d Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 01:33:53 +0200 Subject: [PATCH 50/76] cpu and gpu --- docker/Dockerfile | 20 ++++++++------------ docker/Dockerfile.gpu | 20 ++++++++------------ 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index fc2a5cab1..b719d7f52 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -33,18 +33,14 @@ RUN pip install --no-cache-dir \ COPY ./requirements.txt /tmp/ RUN pip install --no-cache-dir -r /tmp/requirements.txt -# Install all widget bits via pip -RUN pip install --no-cache-dir \ - ipywidgets \ - jupyterlab_widgets \ - widgetsnbextension - -# Enable classic-notebook widget nbextension -RUN jupyter nbextension install --py widgetsnbextension --sys-prefix && \ - jupyter nbextension enable --py widgetsnbextension --sys-prefix - -# Enable the server extension for JupyterLab & VS Code -RUN jupyter server extension enable --py jupyterlab_widgets --sys-prefix +# Enable Jupyter widget support +RUN pip install --no-cache-dir ipywidgets + +# For JupyterLab (inside the container), install the widget manager +RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager \ + --no-build && \ + jupyter lab build && \ + jupyter labextension enable @jupyter-widgets/jupyterlab-manager EXPOSE 8888 CMD ["start-notebook.sh"] \ No newline at end of file diff --git a/docker/Dockerfile.gpu b/docker/Dockerfile.gpu index 493da8212..2c763ffae 100644 --- a/docker/Dockerfile.gpu +++ b/docker/Dockerfile.gpu @@ -29,18 +29,14 @@ RUN pip install --no-cache-dir \ COPY ./requirements.txt /tmp/ RUN pip install --no-cache-dir -r /tmp/requirements.txt -# Install all widget bits via pip -RUN pip install --no-cache-dir \ - ipywidgets \ - jupyterlab_widgets \ - widgetsnbextension - -# Enable classic-notebook widget nbextension -RUN jupyter nbextension install --py widgetsnbextension --sys-prefix && \ - jupyter nbextension enable --py widgetsnbextension --sys-prefix - -# Enable the server extension for JupyterLab & VS Code -RUN jupyter server extension enable --py jupyterlab_widgets --sys-prefix +# Enable Jupyter widget support +RUN pip install --no-cache-dir ipywidgets + +# For JupyterLab (inside the container), install the widget manager +RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager \ + --no-build && \ + jupyter lab build && \ + jupyter labextension enable @jupyter-widgets/jupyterlab-manager EXPOSE 8888 CMD ["start-notebook.sh"] From 60f9a146069ec0f6124f2ba26110c08d7e508680 Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 02:30:37 +0200 Subject: [PATCH 51/76] cpu and gpu --- docker/Dockerfile | 5 ++--- docker/Dockerfile.gpu | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b719d7f52..9fea2e580 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -39,8 +39,7 @@ RUN pip install --no-cache-dir ipywidgets # For JupyterLab (inside the container), install the widget manager RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager \ --no-build && \ - jupyter lab build && \ - jupyter labextension enable @jupyter-widgets/jupyterlab-manager - + jupyter lab build + EXPOSE 8888 CMD ["start-notebook.sh"] \ No newline at end of file diff --git a/docker/Dockerfile.gpu b/docker/Dockerfile.gpu index 2c763ffae..01da469e8 100644 --- a/docker/Dockerfile.gpu +++ b/docker/Dockerfile.gpu @@ -35,8 +35,7 @@ RUN pip install --no-cache-dir ipywidgets # For JupyterLab (inside the container), install the widget manager RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager \ --no-build && \ - jupyter lab build && \ - jupyter labextension enable @jupyter-widgets/jupyterlab-manager - + jupyter lab build + EXPOSE 8888 CMD ["start-notebook.sh"] From 1a7edd4d2da594746fd738dda7b669f31f72dcc3 Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 09:40:20 +0200 Subject: [PATCH 52/76] cpu and gpu --- .github/workflows/cleanup-ghcr.yml | 57 ++++++++++++++++++++++++++++++ docker/Dockerfile | 5 --- docker/Dockerfile.gpu | 5 --- 3 files changed, 57 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/cleanup-ghcr.yml diff --git a/.github/workflows/cleanup-ghcr.yml b/.github/workflows/cleanup-ghcr.yml new file mode 100644 index 000000000..412869fd1 --- /dev/null +++ b/.github/workflows/cleanup-ghcr.yml @@ -0,0 +1,57 @@ +name: "Prune old GHCR images" + +on: + # 1) Run daily at 02:00 UTC + schedule: + - cron: '0 2 * * *' + # 2) Also run on every push to main (or your default branch) + push: + branches: + - docker + +permissions: + contents: read # for listing versions + packages: write # for deleting package versions + +jobs: + prune: + runs-on: ubuntu-latest + + steps: + - name: Install jq and GitHub CLI + run: | + sudo apt-get update + sudo apt-get install -y jq + curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \ + | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] \ + https://cli.github.com/packages stable main" \ + | sudo tee /etc/apt/sources.list.d/github-cli.list + sudo apt-get update + sudo apt-get install -y gh + + - name: Authenticate gh + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "$GITHUB_TOKEN" | gh auth login --with-token + + - name: Delete non-latest versions + env: + OWNER: ${{ github.repository_owner }} + IMAGE: deep-learning-crash-course + run: | + gh api -H "Accept: application/vnd.github.v3+json" \ + /users/$OWNER/packages/container/$IMAGE/versions \ + | jq -c '.[]' \ + | while read version; do + id=$(echo "$version" | jq -r '.id') + tags=$(echo "$version" | jq -r '.metadata.container.tags[]') + if [[ ! " $tags " =~ " latest " ]]; then + echo "Deleting version $id (tags: $tags)" + gh api -X DELETE \ + -H "Accept: application/vnd.github.v3+json" \ + /users/$OWNER/packages/container/$IMAGE/versions/$id + fi + done diff --git a/docker/Dockerfile b/docker/Dockerfile index 9fea2e580..4e02c61d3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -36,10 +36,5 @@ RUN pip install --no-cache-dir -r /tmp/requirements.txt # Enable Jupyter widget support RUN pip install --no-cache-dir ipywidgets -# For JupyterLab (inside the container), install the widget manager -RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager \ - --no-build && \ - jupyter lab build - EXPOSE 8888 CMD ["start-notebook.sh"] \ No newline at end of file diff --git a/docker/Dockerfile.gpu b/docker/Dockerfile.gpu index 01da469e8..455a04e35 100644 --- a/docker/Dockerfile.gpu +++ b/docker/Dockerfile.gpu @@ -32,10 +32,5 @@ RUN pip install --no-cache-dir -r /tmp/requirements.txt # Enable Jupyter widget support RUN pip install --no-cache-dir ipywidgets -# For JupyterLab (inside the container), install the widget manager -RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager \ - --no-build && \ - jupyter lab build - EXPOSE 8888 CMD ["start-notebook.sh"] From e31836f2f7db48b8c828a6e56e2a9c5d2303c207 Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 09:47:03 +0200 Subject: [PATCH 53/76] cpu and gpu --- .github/workflows/cleanup-ghcr.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/cleanup-ghcr.yml b/.github/workflows/cleanup-ghcr.yml index 412869fd1..dfb03bf98 100644 --- a/.github/workflows/cleanup-ghcr.yml +++ b/.github/workflows/cleanup-ghcr.yml @@ -31,11 +31,6 @@ jobs: sudo apt-get update sudo apt-get install -y gh - - name: Authenticate gh - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - echo "$GITHUB_TOKEN" | gh auth login --with-token - name: Delete non-latest versions env: From a3aa8b3ee6f2923b19569c8029fa5177a13c7318 Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 09:59:59 +0200 Subject: [PATCH 54/76] cpu and gpu --- .github/workflows/cleanup-ghcr.yml | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/.github/workflows/cleanup-ghcr.yml b/.github/workflows/cleanup-ghcr.yml index dfb03bf98..6fdc99fab 100644 --- a/.github/workflows/cleanup-ghcr.yml +++ b/.github/workflows/cleanup-ghcr.yml @@ -1,44 +1,38 @@ name: "Prune old GHCR images" on: - # 1) Run daily at 02:00 UTC schedule: - - cron: '0 2 * * *' - # 2) Also run on every push to main (or your default branch) + - cron: '0 2 * * *' # daily at 02:00 UTC push: - branches: - - docker - + branches: [ docker ] permissions: - contents: read # for listing versions - packages: write # for deleting package versions + contents: read + packages: write jobs: prune: runs-on: ubuntu-latest steps: - - name: Install jq and GitHub CLI + - name: Install dependencies run: | sudo apt-get update sudo apt-get install -y jq curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \ | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg - echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] \ - https://cli.github.com/packages stable main" \ - | sudo tee /etc/apt/sources.list.d/github-cli.list + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" \ + | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null sudo apt-get update sudo apt-get install -y gh - - name: Delete non-latest versions env: OWNER: ${{ github.repository_owner }} IMAGE: deep-learning-crash-course run: | + echo "Fetching all versions of $OWNER/$IMAGE…" gh api -H "Accept: application/vnd.github.v3+json" \ - /users/$OWNER/packages/container/$IMAGE/versions \ + /orgs/$OWNER/packages/container/$IMAGE/versions \ | jq -c '.[]' \ | while read version; do id=$(echo "$version" | jq -r '.id') @@ -47,6 +41,6 @@ jobs: echo "Deleting version $id (tags: $tags)" gh api -X DELETE \ -H "Accept: application/vnd.github.v3+json" \ - /users/$OWNER/packages/container/$IMAGE/versions/$id + /orgs/$OWNER/packages/container/$IMAGE/versions/$id fi done From 63b12518bb4a5355f94809f99d16efb1c67c9a0d Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 10:21:25 +0200 Subject: [PATCH 55/76] cpu and gpu --- .github/workflows/cleanup-ghcr.yml | 34 +++++++++++++++++------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/.github/workflows/cleanup-ghcr.yml b/.github/workflows/cleanup-ghcr.yml index 6fdc99fab..0bba06f60 100644 --- a/.github/workflows/cleanup-ghcr.yml +++ b/.github/workflows/cleanup-ghcr.yml @@ -25,22 +25,26 @@ jobs: sudo apt-get update sudo apt-get install -y gh - - name: Delete non-latest versions + - name: Delete all but the newest version env: OWNER: ${{ github.repository_owner }} IMAGE: deep-learning-crash-course run: | - echo "Fetching all versions of $OWNER/$IMAGE…" - gh api -H "Accept: application/vnd.github.v3+json" \ - /orgs/$OWNER/packages/container/$IMAGE/versions \ - | jq -c '.[]' \ - | while read version; do - id=$(echo "$version" | jq -r '.id') - tags=$(echo "$version" | jq -r '.metadata.container.tags[]') - if [[ ! " $tags " =~ " latest " ]]; then - echo "Deleting version $id (tags: $tags)" - gh api -X DELETE \ - -H "Accept: application/vnd.github.v3+json" \ - /orgs/$OWNER/packages/container/$IMAGE/versions/$id - fi - done + echo "Fetching all versions of $OWNER/$IMAGE..." + versions=$(gh api --paginate -H "Accept: application/vnd.github.v3+json" \ + /orgs/$OWNER/packages/container/$IMAGE/versions) + # find the newest by creation timestamp + newest_id=$(echo "$versions" \ + | jq -r 'sort_by(.created_at) | reverse | .[0].id') + echo "🛡 Keeping version $newest_id (most recent)" + # loop through all and delete those that aren't the newest + echo "$versions" | jq -c '.[]' | while read ver; do + id=$(echo "$ver" | jq -r '.id') + if [ "$id" != "$newest_id" ]; then + echo "Deleting version $id" + gh api -X DELETE \ + -H "Accept: application/vnd.github.v3+json" \ + /orgs/$OWNER/packages/container/$IMAGE/versions/$id + fi + done + From f235371624ba284b282a52abdb74cc8ed83dcf5e Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 10:25:30 +0200 Subject: [PATCH 56/76] cpu and gpu --- .github/workflows/cleanup-ghcr.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cleanup-ghcr.yml b/.github/workflows/cleanup-ghcr.yml index 0bba06f60..efa1d4ddd 100644 --- a/.github/workflows/cleanup-ghcr.yml +++ b/.github/workflows/cleanup-ghcr.yml @@ -29,15 +29,14 @@ jobs: env: OWNER: ${{ github.repository_owner }} IMAGE: deep-learning-crash-course + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | echo "Fetching all versions of $OWNER/$IMAGE..." versions=$(gh api --paginate -H "Accept: application/vnd.github.v3+json" \ /orgs/$OWNER/packages/container/$IMAGE/versions) - # find the newest by creation timestamp newest_id=$(echo "$versions" \ | jq -r 'sort_by(.created_at) | reverse | .[0].id') echo "🛡 Keeping version $newest_id (most recent)" - # loop through all and delete those that aren't the newest echo "$versions" | jq -c '.[]' | while read ver; do id=$(echo "$ver" | jq -r '.id') if [ "$id" != "$newest_id" ]; then @@ -48,3 +47,4 @@ jobs: fi done + From d5932bbd8d02d80d1802f52755a4862187b3d7fc Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 10:32:26 +0200 Subject: [PATCH 57/76] cpu and gpu --- .github/workflows/cleanup-ghcr.yml | 41 ++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/.github/workflows/cleanup-ghcr.yml b/.github/workflows/cleanup-ghcr.yml index efa1d4ddd..2485beba0 100644 --- a/.github/workflows/cleanup-ghcr.yml +++ b/.github/workflows/cleanup-ghcr.yml @@ -25,26 +25,39 @@ jobs: sudo apt-get update sudo apt-get install -y gh + - name: Delete all but the newest version env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} OWNER: ${{ github.repository_owner }} IMAGE: deep-learning-crash-course - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - echo "Fetching all versions of $OWNER/$IMAGE..." - versions=$(gh api --paginate -H "Accept: application/vnd.github.v3+json" \ - /orgs/$OWNER/packages/container/$IMAGE/versions) + set -euo pipefail + + echo "Fetching all versions of $OWNER/$IMAGE…" + versions=$(gh api --paginate \ + -H "Accept: application/vnd.github.v3+json" \ + /orgs/$OWNER/packages/container/$IMAGE/versions) + newest_id=$(echo "$versions" \ - | jq -r 'sort_by(.created_at) | reverse | .[0].id') + | jq -r 'sort_by(.created_at) | reverse | .[0].id') echo "🛡 Keeping version $newest_id (most recent)" - echo "$versions" | jq -c '.[]' | while read ver; do - id=$(echo "$ver" | jq -r '.id') - if [ "$id" != "$newest_id" ]; then - echo "Deleting version $id" - gh api -X DELETE \ - -H "Accept: application/vnd.github.v3+json" \ - /orgs/$OWNER/packages/container/$IMAGE/versions/$id + + echo "$versions" | jq -c '.[]' | while read version; do + id=$(echo "$version" | jq -r '.id') + if [[ "$id" != "$newest_id" ]]; then + echo "→ Deleting version $id" + # retry up to 3 times on transient errors + for attempt in 1 2 3; do + if gh api -X DELETE \ + -H "Accept: application/vnd.github.v3+json" \ + /orgs/$OWNER/packages/container/$IMAGE/versions/$id; then + echo " ✅ Deleted $id" + break + else + echo " ⚠️ Attempt $attempt failed, retrying in $((5*attempt))s…" + sleep $((5*attempt)) + fi + done fi done - - From ca5208b2deeaf12ec53fdd298332c687d775d708 Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 10:39:43 +0200 Subject: [PATCH 58/76] cpu and gpu --- .github/workflows/cleanup-ghcr.yml | 63 ++++++++++++++++-------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/.github/workflows/cleanup-ghcr.yml b/.github/workflows/cleanup-ghcr.yml index 2485beba0..02bee75fb 100644 --- a/.github/workflows/cleanup-ghcr.yml +++ b/.github/workflows/cleanup-ghcr.yml @@ -5,6 +5,7 @@ on: - cron: '0 2 * * *' # daily at 02:00 UTC push: branches: [ docker ] + permissions: contents: read packages: write @@ -25,39 +26,41 @@ jobs: sudo apt-get update sudo apt-get install -y gh - - - name: Delete all but the newest version + - name: Delete all but the newest version for CPU & GPU images env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OWNER: ${{ github.repository_owner }} - IMAGE: deep-learning-crash-course + OWNER: ${{ github.repository_owner }} run: | set -euo pipefail - echo "Fetching all versions of $OWNER/$IMAGE…" - versions=$(gh api --paginate \ - -H "Accept: application/vnd.github.v3+json" \ - /orgs/$OWNER/packages/container/$IMAGE/versions) - - newest_id=$(echo "$versions" \ - | jq -r 'sort_by(.created_at) | reverse | .[0].id') - echo "🛡 Keeping version $newest_id (most recent)" - - echo "$versions" | jq -c '.[]' | while read version; do - id=$(echo "$version" | jq -r '.id') - if [[ "$id" != "$newest_id" ]]; then - echo "→ Deleting version $id" - # retry up to 3 times on transient errors - for attempt in 1 2 3; do - if gh api -X DELETE \ - -H "Accept: application/vnd.github.v3+json" \ - /orgs/$OWNER/packages/container/$IMAGE/versions/$id; then - echo " ✅ Deleted $id" - break - else - echo " ⚠️ Attempt $attempt failed, retrying in $((5*attempt))s…" - sleep $((5*attempt)) - fi - done - fi + for IMAGE in deep-learning-crash-course deep-learning-crash-course-gpu; do + echo + echo "🔍 Processing package: $OWNER/$IMAGE" + versions=$(gh api --paginate \ + -H "Accept: application/vnd.github.v3+json" \ + /orgs/$OWNER/packages/container/$IMAGE/versions) + + # pick most recent by creation date + newest_id=$(echo "$versions" \ + | jq -r 'sort_by(.created_at) | reverse | .[0].id') + echo "🛡 Keeping version $newest_id for $IMAGE" + + # delete every version except the newest + echo "$versions" | jq -c '.[]' | while read version; do + id=$(echo "$version" | jq -r '.id') + if [[ "$id" != "$newest_id" ]]; then + echo "→ Deleting version $id of $IMAGE" + for attempt in 1 2 3; do + if gh api -X DELETE \ + -H "Accept: application/vnd.github.v3+json" \ + /orgs/$OWNER/packages/container/$IMAGE/versions/$id; then + echo " ✅ Deleted $id" + break + else + echo " ⚠️ Attempt $attempt failed, retrying in $((5*attempt))s…" + sleep $((5*attempt)) + fi + done + fi + done done From c87d5a792b2e4ad5ca3a0e4e4bc0bfd6b8e32f7d Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 11:12:14 +0200 Subject: [PATCH 59/76] cpu and gpu --- docker/Dockerfile | 3 --- docker/Dockerfile.gpu | 3 --- 2 files changed, 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4e02c61d3..5cb75638d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -33,8 +33,5 @@ RUN pip install --no-cache-dir \ COPY ./requirements.txt /tmp/ RUN pip install --no-cache-dir -r /tmp/requirements.txt -# Enable Jupyter widget support -RUN pip install --no-cache-dir ipywidgets - EXPOSE 8888 CMD ["start-notebook.sh"] \ No newline at end of file diff --git a/docker/Dockerfile.gpu b/docker/Dockerfile.gpu index 455a04e35..0c08699b3 100644 --- a/docker/Dockerfile.gpu +++ b/docker/Dockerfile.gpu @@ -29,8 +29,5 @@ RUN pip install --no-cache-dir \ COPY ./requirements.txt /tmp/ RUN pip install --no-cache-dir -r /tmp/requirements.txt -# Enable Jupyter widget support -RUN pip install --no-cache-dir ipywidgets - EXPOSE 8888 CMD ["start-notebook.sh"] From 316820bb4ba6abade91119fb20a4497e79fadeb8 Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 11:51:05 +0200 Subject: [PATCH 60/76] trigger cleanup after publishing container --- .github/workflows/cleanup-ghcr.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cleanup-ghcr.yml b/.github/workflows/cleanup-ghcr.yml index 02bee75fb..a0046ca1e 100644 --- a/.github/workflows/cleanup-ghcr.yml +++ b/.github/workflows/cleanup-ghcr.yml @@ -1,10 +1,13 @@ name: "Prune old GHCR images" on: - schedule: - - cron: '0 2 * * *' # daily at 02:00 UTC - push: - branches: [ docker ] + registry_package: + types: [ published ] + package_type: container + # schedule: + # - cron: '0 2 * * *' # daily at 02:00 UTC + # push: + # branches: [ docker ] permissions: contents: read From 0b249c61a42c23e0362bf49e2b2b9ddcddde6445 Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 12:16:35 +0200 Subject: [PATCH 61/76] Update cleanup-ghcr.yml --- .github/workflows/cleanup-ghcr.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cleanup-ghcr.yml b/.github/workflows/cleanup-ghcr.yml index a0046ca1e..4daef546c 100644 --- a/.github/workflows/cleanup-ghcr.yml +++ b/.github/workflows/cleanup-ghcr.yml @@ -4,10 +4,6 @@ on: registry_package: types: [ published ] package_type: container - # schedule: - # - cron: '0 2 * * *' # daily at 02:00 UTC - # push: - # branches: [ docker ] permissions: contents: read @@ -67,3 +63,8 @@ jobs: fi done done + + # schedule: + # - cron: '0 2 * * *' # daily at 02:00 UTC + # push: + # branches: [ docker ] \ No newline at end of file From 29b0ba979d45463f6d0e44e03570f085eba3585c Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 12:43:35 +0200 Subject: [PATCH 62/76] json settings for vscode --- .github/workflows/cleanup-ghcr.yml | 11 +++++------ docker/.vscode/settings.json | 11 +++++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 docker/.vscode/settings.json diff --git a/.github/workflows/cleanup-ghcr.yml b/.github/workflows/cleanup-ghcr.yml index 4daef546c..688ed3b43 100644 --- a/.github/workflows/cleanup-ghcr.yml +++ b/.github/workflows/cleanup-ghcr.yml @@ -1,6 +1,10 @@ name: "Prune old GHCR images" on: + schedule: + - cron: '0 2 * * *' # daily at 02:00 UTC + push: + branches: [ docker ] registry_package: types: [ published ] package_type: container @@ -62,9 +66,4 @@ jobs: done fi done - done - - # schedule: - # - cron: '0 2 * * *' # daily at 02:00 UTC - # push: - # branches: [ docker ] \ No newline at end of file + done \ No newline at end of file diff --git a/docker/.vscode/settings.json b/docker/.vscode/settings.json new file mode 100644 index 000000000..7693d3b1c --- /dev/null +++ b/docker/.vscode/settings.json @@ -0,0 +1,11 @@ +{ + // Allow the VS Code widget renderer to fetch JS directly + "jupyter.widgetScriptSources": [ + "localhost:8888", + "jsdelivr.com", + "unpkg.com" + ], + + // Make sure VS Code always uses the container’s Python + "python.defaultInterpreterPath": "/opt/conda/bin/python" +} From 2d4c9b7051ad96917724ee0d371dfe7300c417d8 Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 12:46:25 +0200 Subject: [PATCH 63/76] Update docker-publish.yml --- .github/workflows/docker-publish.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index ed1f66ad0..502554bd3 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -9,6 +9,7 @@ on: - 'docker/requirements.txt' - 'docker/docker-compose.yml' - '.github/workflows/docker-publish.yml' + - 'docker/.vscode/settings.json' pull_request: branches: [ docker ] paths: @@ -17,6 +18,8 @@ on: - 'docker/requirements.txt' - 'docker/docker-compose.yml' - '.github/workflows/docker-publish.yml' + - 'docker/.vscode/settings.json' + env: REGISTRY: ghcr.io From 491d5430c2cde50639c609599705a40f9a256e5b Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 13:13:23 +0200 Subject: [PATCH 64/76] tidy up --- .github/workflows/cleanup-ghcr.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cleanup-ghcr.yml b/.github/workflows/cleanup-ghcr.yml index 688ed3b43..a13d8e7af 100644 --- a/.github/workflows/cleanup-ghcr.yml +++ b/.github/workflows/cleanup-ghcr.yml @@ -7,7 +7,6 @@ on: branches: [ docker ] registry_package: types: [ published ] - package_type: container permissions: contents: read From f0310ec03364e24606f11d25a794d5604bf1eb00 Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 13:22:31 +0200 Subject: [PATCH 65/76] delete all --- ...nup-ghcr.yml => cleanup-ghcr.yml.disabled} | 0 .github/workflows/prune-all.yml | 58 +++++++++++++++++++ 2 files changed, 58 insertions(+) rename .github/workflows/{cleanup-ghcr.yml => cleanup-ghcr.yml.disabled} (100%) create mode 100644 .github/workflows/prune-all.yml diff --git a/.github/workflows/cleanup-ghcr.yml b/.github/workflows/cleanup-ghcr.yml.disabled similarity index 100% rename from .github/workflows/cleanup-ghcr.yml rename to .github/workflows/cleanup-ghcr.yml.disabled diff --git a/.github/workflows/prune-all.yml b/.github/workflows/prune-all.yml new file mode 100644 index 000000000..671596944 --- /dev/null +++ b/.github/workflows/prune-all.yml @@ -0,0 +1,58 @@ +name: "Prune all GHCR image versions" + +on: + push: + branches: [ docker ] + +permissions: + contents: read + packages: write + +jobs: + prune: + runs-on: ubuntu-latest + + steps: + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y jq + curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \ + | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" \ + | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null + sudo apt-get update + sudo apt-get install -y gh + + - name: Delete **all** versions of CPU & GPU images + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OWNER: ${{ github.repository_owner }} + run: | + set -euo pipefail + + for IMAGE in deep-learning-crash-course deep-learning-crash-course-gpu; do + echo + echo "🔍 Processing package: $OWNER/$IMAGE" + + # Fetch all versions (paginated) + versions=$(gh api --paginate \ + -H "Accept: application/vnd.github.v3+json" \ + /orgs/$OWNER/packages/container/$IMAGE/versions) + + # Delete every version ID + echo "$versions" | jq -r '.[].id' | while read id; do + echo "→ Deleting version $id of $IMAGE" + for attempt in 1 2 3; do + if gh api -X DELETE \ + -H "Accept: application/vnd.github.v3+json" \ + /orgs/$OWNER/packages/container/$IMAGE/versions/$id; then + echo " ✅ Deleted $id" + break + else + echo " ⚠️ Attempt $attempt failed, retrying in $((5*attempt))s…" + sleep $((5*attempt)) + fi + done + done + done From 61ad58f44f909bea8bc204a51408b237435801a4 Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 13:29:32 +0200 Subject: [PATCH 66/76] clean all --- .github/workflows/prune-all.yml | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/prune-all.yml b/.github/workflows/prune-all.yml index 671596944..da7089322 100644 --- a/.github/workflows/prune-all.yml +++ b/.github/workflows/prune-all.yml @@ -1,4 +1,4 @@ -name: "Prune all GHCR image versions" +name: "Prune old GHCR images" on: push: @@ -24,7 +24,7 @@ jobs: sudo apt-get update sudo apt-get install -y gh - - name: Delete **all** versions of CPU & GPU images + - name: Delete all but the newest version for CPU & GPU images env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} OWNER: ${{ github.repository_owner }} @@ -34,25 +34,24 @@ jobs: for IMAGE in deep-learning-crash-course deep-learning-crash-course-gpu; do echo echo "🔍 Processing package: $OWNER/$IMAGE" - - # Fetch all versions (paginated) versions=$(gh api --paginate \ -H "Accept: application/vnd.github.v3+json" \ /orgs/$OWNER/packages/container/$IMAGE/versions) - # Delete every version ID - echo "$versions" | jq -r '.[].id' | while read id; do + # delete every version except the newest + echo "$versions" | jq -c '.[]' | while read version; do + id=$(echo "$version" | jq -r '.id') echo "→ Deleting version $id of $IMAGE" for attempt in 1 2 3; do - if gh api -X DELETE \ - -H "Accept: application/vnd.github.v3+json" \ - /orgs/$OWNER/packages/container/$IMAGE/versions/$id; then + if gh api -X DELETE \ + -H "Accept: application/vnd.github.v3+json" \ + /orgs/$OWNER/packages/container/$IMAGE/versions/$id; then echo " ✅ Deleted $id" break - else + else echo " ⚠️ Attempt $attempt failed, retrying in $((5*attempt))s…" sleep $((5*attempt)) - fi + fi done done - done + done \ No newline at end of file From fea807b39b3e4adb24b52d8b9ceba7551321b015 Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 13:36:12 +0200 Subject: [PATCH 67/76] rebuild all --- ...nup-ghcr.yml.disabled => cleanup-ghcr.yml} | 0 .../{prune-all.yml => prune-all.yml.disabled} | 23 ++++++++++--------- docker/.vscode/settings.json | 1 - 3 files changed, 12 insertions(+), 12 deletions(-) rename .github/workflows/{cleanup-ghcr.yml.disabled => cleanup-ghcr.yml} (100%) rename .github/workflows/{prune-all.yml => prune-all.yml.disabled} (74%) diff --git a/.github/workflows/cleanup-ghcr.yml.disabled b/.github/workflows/cleanup-ghcr.yml similarity index 100% rename from .github/workflows/cleanup-ghcr.yml.disabled rename to .github/workflows/cleanup-ghcr.yml diff --git a/.github/workflows/prune-all.yml b/.github/workflows/prune-all.yml.disabled similarity index 74% rename from .github/workflows/prune-all.yml rename to .github/workflows/prune-all.yml.disabled index da7089322..671596944 100644 --- a/.github/workflows/prune-all.yml +++ b/.github/workflows/prune-all.yml.disabled @@ -1,4 +1,4 @@ -name: "Prune old GHCR images" +name: "Prune all GHCR image versions" on: push: @@ -24,7 +24,7 @@ jobs: sudo apt-get update sudo apt-get install -y gh - - name: Delete all but the newest version for CPU & GPU images + - name: Delete **all** versions of CPU & GPU images env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} OWNER: ${{ github.repository_owner }} @@ -34,24 +34,25 @@ jobs: for IMAGE in deep-learning-crash-course deep-learning-crash-course-gpu; do echo echo "🔍 Processing package: $OWNER/$IMAGE" + + # Fetch all versions (paginated) versions=$(gh api --paginate \ -H "Accept: application/vnd.github.v3+json" \ /orgs/$OWNER/packages/container/$IMAGE/versions) - # delete every version except the newest - echo "$versions" | jq -c '.[]' | while read version; do - id=$(echo "$version" | jq -r '.id') + # Delete every version ID + echo "$versions" | jq -r '.[].id' | while read id; do echo "→ Deleting version $id of $IMAGE" for attempt in 1 2 3; do - if gh api -X DELETE \ - -H "Accept: application/vnd.github.v3+json" \ - /orgs/$OWNER/packages/container/$IMAGE/versions/$id; then + if gh api -X DELETE \ + -H "Accept: application/vnd.github.v3+json" \ + /orgs/$OWNER/packages/container/$IMAGE/versions/$id; then echo " ✅ Deleted $id" break - else + else echo " ⚠️ Attempt $attempt failed, retrying in $((5*attempt))s…" sleep $((5*attempt)) - fi + fi done done - done \ No newline at end of file + done diff --git a/docker/.vscode/settings.json b/docker/.vscode/settings.json index 7693d3b1c..275214867 100644 --- a/docker/.vscode/settings.json +++ b/docker/.vscode/settings.json @@ -5,7 +5,6 @@ "jsdelivr.com", "unpkg.com" ], - // Make sure VS Code always uses the container’s Python "python.defaultInterpreterPath": "/opt/conda/bin/python" } From 94489a4bb1c782bd1b32441150a7c9933246cef2 Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 13:57:54 +0200 Subject: [PATCH 68/76] clean up modified --- .github/workflows/cleanup-ghcr.yml | 25 +++++++++++-------------- docker/README.md | 2 +- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/.github/workflows/cleanup-ghcr.yml b/.github/workflows/cleanup-ghcr.yml index a13d8e7af..07fd74ce8 100644 --- a/.github/workflows/cleanup-ghcr.yml +++ b/.github/workflows/cleanup-ghcr.yml @@ -28,7 +28,7 @@ jobs: sudo apt-get update sudo apt-get install -y gh - - name: Delete all but the newest version for CPU & GPU images + - name: Delete all non-`:latest` versions env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} OWNER: ${{ github.repository_owner }} @@ -42,16 +42,14 @@ jobs: -H "Accept: application/vnd.github.v3+json" \ /orgs/$OWNER/packages/container/$IMAGE/versions) - # pick most recent by creation date - newest_id=$(echo "$versions" \ - | jq -r 'sort_by(.created_at) | reverse | .[0].id') - echo "🛡 Keeping version $newest_id for $IMAGE" + # Only delete versions *not* tagged "latest" + echo "$versions" \ + | jq -c '.[] | select((.metadata.container.tags // []) | index("latest") | not)' \ + | while read version; do - # delete every version except the newest - echo "$versions" | jq -c '.[]' | while read version; do - id=$(echo "$version" | jq -r '.id') - if [[ "$id" != "$newest_id" ]]; then - echo "→ Deleting version $id of $IMAGE" + id=$(echo "$version" | jq -r '.id') + tags=$(echo "$version" | jq -r '.metadata.container.tags[]') + echo "→ Deleting version $id (tags: $tags)" for attempt in 1 2 3; do if gh api -X DELETE \ -H "Accept: application/vnd.github.v3+json" \ @@ -59,10 +57,9 @@ jobs: echo " ✅ Deleted $id" break else - echo " ⚠️ Attempt $attempt failed, retrying in $((5*attempt))s…" + echo " ⚠️ Attempt $attempt failed, retrying…" sleep $((5*attempt)) fi done - fi - done - done \ No newline at end of file + done + done diff --git a/docker/README.md b/docker/README.md index bcec0c849..c44d8b7ab 100644 --- a/docker/README.md +++ b/docker/README.md @@ -71,4 +71,4 @@ After startup, copy the URL with token (e.g., http://127.0.0.1:8888/lab?token= 5. **Open Folder** → `/home/jovyan/work` and **Select Kernel** → `/opt/conda/bin/python` (Python 3.11). - 6. Open any `.ipynb` and run cells. \ No newline at end of file + 6. Open any `.ipynb` and run cells. If `ipywidgets` fails, Reload window. \ No newline at end of file From 2205f7185d2267f46d34e92298e688c808aad5a5 Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 14:04:49 +0200 Subject: [PATCH 69/76] rebuild --- .github/workflows/cleanup-ghcr.yml | 38 +++++++++++++--------------- .github/workflows/docker-publish.yml | 2 -- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/.github/workflows/cleanup-ghcr.yml b/.github/workflows/cleanup-ghcr.yml index 07fd74ce8..14c63b45c 100644 --- a/.github/workflows/cleanup-ghcr.yml +++ b/.github/workflows/cleanup-ghcr.yml @@ -28,7 +28,7 @@ jobs: sudo apt-get update sudo apt-get install -y gh - - name: Delete all non-`:latest` versions + - name: Re-tag newest as “latest” and prune the rest env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} OWNER: ${{ github.repository_owner }} @@ -37,29 +37,27 @@ jobs: for IMAGE in deep-learning-crash-course deep-learning-crash-course-gpu; do echo - echo "🔍 Processing package: $OWNER/$IMAGE" + echo "🔍 Processing $OWNER/$IMAGE" versions=$(gh api --paginate \ -H "Accept: application/vnd.github.v3+json" \ /orgs/$OWNER/packages/container/$IMAGE/versions) - # Only delete versions *not* tagged "latest" - echo "$versions" \ - | jq -c '.[] | select((.metadata.container.tags // []) | index("latest") | not)' \ - | while read version; do + # pick the newest by creation date + newest_id=$(echo "$versions" \ + | jq -r 'sort_by(.created_at) | last | .id') + echo "🛡 Retagging version $newest_id as latest" + gh api -X POST \ + -H "Accept: application/vnd.github.v3+json" \ + /orgs/$OWNER/packages/container/$IMAGE/versions/$newest_id/tags \ + -f name=latest - id=$(echo "$version" | jq -r '.id') - tags=$(echo "$version" | jq -r '.metadata.container.tags[]') - echo "→ Deleting version $id (tags: $tags)" - for attempt in 1 2 3; do - if gh api -X DELETE \ - -H "Accept: application/vnd.github.v3+json" \ - /orgs/$OWNER/packages/container/$IMAGE/versions/$id; then - echo " ✅ Deleted $id" - break - else - echo " ⚠️ Attempt $attempt failed, retrying…" - sleep $((5*attempt)) - fi + # delete all other versions + echo "$versions" \ + | jq -r --arg keep "$newest_id" '.[] | select(.id != ($keep|tonumber)) | .id' \ + | while read id; do + echo "→ Deleting version $id" + gh api -X DELETE \ + -H "Accept: application/vnd.github.v3+json" \ + /orgs/$OWNER/packages/container/$IMAGE/versions/$id done - done done diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 502554bd3..bd5a9e649 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -9,7 +9,6 @@ on: - 'docker/requirements.txt' - 'docker/docker-compose.yml' - '.github/workflows/docker-publish.yml' - - 'docker/.vscode/settings.json' pull_request: branches: [ docker ] paths: @@ -18,7 +17,6 @@ on: - 'docker/requirements.txt' - 'docker/docker-compose.yml' - '.github/workflows/docker-publish.yml' - - 'docker/.vscode/settings.json' env: From bbfb87419d425345f24c8db64c20e0da224f5357 Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 14:22:39 +0200 Subject: [PATCH 70/76] cpu and gpu --- .github/workflows/docker-publish.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index bd5a9e649..ed1f66ad0 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -18,7 +18,6 @@ on: - 'docker/docker-compose.yml' - '.github/workflows/docker-publish.yml' - env: REGISTRY: ghcr.io IMAGE_NAME: deep-learning-crash-course From 7c8cc3f6e250bf8f3b8fe87e9a560a895e3390ab Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 14:23:43 +0200 Subject: [PATCH 71/76] cpu and gpu --- .github/workflows/cleanup-ghcr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cleanup-ghcr.yml b/.github/workflows/cleanup-ghcr.yml index 14c63b45c..10b6ad69e 100644 --- a/.github/workflows/cleanup-ghcr.yml +++ b/.github/workflows/cleanup-ghcr.yml @@ -60,4 +60,4 @@ jobs: -H "Accept: application/vnd.github.v3+json" \ /orgs/$OWNER/packages/container/$IMAGE/versions/$id done - done + done \ No newline at end of file From 8d1eeaa5c0c2955fe3c2816083fcc030527765f4 Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 14:24:01 +0200 Subject: [PATCH 72/76] Revert "cpu and gpu" This reverts commit bbfb87419d425345f24c8db64c20e0da224f5357. --- .github/workflows/docker-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index ed1f66ad0..bd5a9e649 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -18,6 +18,7 @@ on: - 'docker/docker-compose.yml' - '.github/workflows/docker-publish.yml' + env: REGISTRY: ghcr.io IMAGE_NAME: deep-learning-crash-course From 9043027135ab26dfc3b7d7626d4955d8c3134175 Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 14:26:17 +0200 Subject: [PATCH 73/76] old cleanup file --- .github/workflows/cleanup-ghcr.yml | 41 +++++++++++++++++------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/.github/workflows/cleanup-ghcr.yml b/.github/workflows/cleanup-ghcr.yml index 10b6ad69e..a13d8e7af 100644 --- a/.github/workflows/cleanup-ghcr.yml +++ b/.github/workflows/cleanup-ghcr.yml @@ -28,7 +28,7 @@ jobs: sudo apt-get update sudo apt-get install -y gh - - name: Re-tag newest as “latest” and prune the rest + - name: Delete all but the newest version for CPU & GPU images env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} OWNER: ${{ github.repository_owner }} @@ -37,27 +37,32 @@ jobs: for IMAGE in deep-learning-crash-course deep-learning-crash-course-gpu; do echo - echo "🔍 Processing $OWNER/$IMAGE" + echo "🔍 Processing package: $OWNER/$IMAGE" versions=$(gh api --paginate \ -H "Accept: application/vnd.github.v3+json" \ /orgs/$OWNER/packages/container/$IMAGE/versions) - # pick the newest by creation date + # pick most recent by creation date newest_id=$(echo "$versions" \ - | jq -r 'sort_by(.created_at) | last | .id') - echo "🛡 Retagging version $newest_id as latest" - gh api -X POST \ - -H "Accept: application/vnd.github.v3+json" \ - /orgs/$OWNER/packages/container/$IMAGE/versions/$newest_id/tags \ - -f name=latest - - # delete all other versions - echo "$versions" \ - | jq -r --arg keep "$newest_id" '.[] | select(.id != ($keep|tonumber)) | .id' \ - | while read id; do - echo "→ Deleting version $id" - gh api -X DELETE \ - -H "Accept: application/vnd.github.v3+json" \ - /orgs/$OWNER/packages/container/$IMAGE/versions/$id + | jq -r 'sort_by(.created_at) | reverse | .[0].id') + echo "🛡 Keeping version $newest_id for $IMAGE" + + # delete every version except the newest + echo "$versions" | jq -c '.[]' | while read version; do + id=$(echo "$version" | jq -r '.id') + if [[ "$id" != "$newest_id" ]]; then + echo "→ Deleting version $id of $IMAGE" + for attempt in 1 2 3; do + if gh api -X DELETE \ + -H "Accept: application/vnd.github.v3+json" \ + /orgs/$OWNER/packages/container/$IMAGE/versions/$id; then + echo " ✅ Deleted $id" + break + else + echo " ⚠️ Attempt $attempt failed, retrying in $((5*attempt))s…" + sleep $((5*attempt)) + fi done + fi + done done \ No newline at end of file From 32314d517de71015a7d0e0d7bca73ae02f37d14b Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 20 May 2025 14:31:37 +0200 Subject: [PATCH 74/76] redo --- .github/workflows/docker-publish.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index bd5a9e649..603ce70c8 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -17,8 +17,7 @@ on: - 'docker/requirements.txt' - 'docker/docker-compose.yml' - '.github/workflows/docker-publish.yml' - - + env: REGISTRY: ghcr.io IMAGE_NAME: deep-learning-crash-course From 7dc32fbaa633d2fb78f8cd18e9ece46153bffe20 Mon Sep 17 00:00:00 2001 From: Carlo Date: Wed, 21 May 2025 10:02:15 +0200 Subject: [PATCH 75/76] disable clean up schedule --- ...nup-ghcr.yml => cleanup-ghcr.yml.disabled} | 0 .github/workflows/prune-all.yml.disabled | 58 ------------------- 2 files changed, 58 deletions(-) rename .github/workflows/{cleanup-ghcr.yml => cleanup-ghcr.yml.disabled} (100%) delete mode 100644 .github/workflows/prune-all.yml.disabled diff --git a/.github/workflows/cleanup-ghcr.yml b/.github/workflows/cleanup-ghcr.yml.disabled similarity index 100% rename from .github/workflows/cleanup-ghcr.yml rename to .github/workflows/cleanup-ghcr.yml.disabled diff --git a/.github/workflows/prune-all.yml.disabled b/.github/workflows/prune-all.yml.disabled deleted file mode 100644 index 671596944..000000000 --- a/.github/workflows/prune-all.yml.disabled +++ /dev/null @@ -1,58 +0,0 @@ -name: "Prune all GHCR image versions" - -on: - push: - branches: [ docker ] - -permissions: - contents: read - packages: write - -jobs: - prune: - runs-on: ubuntu-latest - - steps: - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y jq - curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \ - | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" \ - | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null - sudo apt-get update - sudo apt-get install -y gh - - - name: Delete **all** versions of CPU & GPU images - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OWNER: ${{ github.repository_owner }} - run: | - set -euo pipefail - - for IMAGE in deep-learning-crash-course deep-learning-crash-course-gpu; do - echo - echo "🔍 Processing package: $OWNER/$IMAGE" - - # Fetch all versions (paginated) - versions=$(gh api --paginate \ - -H "Accept: application/vnd.github.v3+json" \ - /orgs/$OWNER/packages/container/$IMAGE/versions) - - # Delete every version ID - echo "$versions" | jq -r '.[].id' | while read id; do - echo "→ Deleting version $id of $IMAGE" - for attempt in 1 2 3; do - if gh api -X DELETE \ - -H "Accept: application/vnd.github.v3+json" \ - /orgs/$OWNER/packages/container/$IMAGE/versions/$id; then - echo " ✅ Deleted $id" - break - else - echo " ⚠️ Attempt $attempt failed, retrying in $((5*attempt))s…" - sleep $((5*attempt)) - fi - done - done - done From b159e3689bcafac5c861d367ebef0b72cc811491 Mon Sep 17 00:00:00 2001 From: Carlo Date: Wed, 28 May 2025 19:42:12 +0200 Subject: [PATCH 76/76] added pygame --- .../workflows/{cleanup-ghcr.yml.disabled => cleanup-ghcr.yml} | 0 docker/requirements.txt | 3 ++- 2 files changed, 2 insertions(+), 1 deletion(-) rename .github/workflows/{cleanup-ghcr.yml.disabled => cleanup-ghcr.yml} (100%) diff --git a/.github/workflows/cleanup-ghcr.yml.disabled b/.github/workflows/cleanup-ghcr.yml similarity index 100% rename from .github/workflows/cleanup-ghcr.yml.disabled rename to .github/workflows/cleanup-ghcr.yml diff --git a/docker/requirements.txt b/docker/requirements.txt index b38641979..cf6af41d4 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -21,4 +21,5 @@ tifffile==2025.5.10 torchmetrics==1.6.1 tqdm==4.67.1 transformers==4.51.3 -warmup_scheduler==0.3 \ No newline at end of file +warmup_scheduler==0.3 +pygame==2.6.1 \ No newline at end of file