Skip to content

Commit a85bfad

Browse files
mathbunnyrumathbunnyru
authored
Merge tags and create multi-platform images (#1789)
* Merge tags and create multi-platform images * Add new files :) * Provide images * Fix * Fix * Use the same naming style as python for the images * Fix docker image rmi command * Use plumbum.FG * Fix * Add missing path * Few merging fixes * Add pushing manifests * Add missing path * Add arch to apply tags * Use platform word * Remove unused step * Merge tags for tensorflow as well * Do not use print * Make merge_tags work when image doesn't exist * Add logs * Add icon * Add tensorflow-notebook workaround * Unify platform usage * Remove unused function * Fix * Update docs
1 parent 494fc9c commit a85bfad

File tree

16 files changed

+376
-142
lines changed

16 files changed

+376
-142
lines changed

‎.github/actions/create-dev-env/action.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ runs:
1717
uses: actions/setup-python@v4
1818
with:
1919
python-version: 3.x
20-
if: ${{ inputs.platform == 'amd64' }}
20+
if: ${{ inputs.platform == 'x86_64' }}
2121

2222
- name: Install Dev Dependencies 📦
2323
run: |

‎.github/actions/download-manifests/action.yml‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ runs:
2626
- name: Download artifact 📥
2727
uses: actions/download-artifact@v3
2828
with:
29-
name: base-notebook-amd64-history_line
29+
name: base-notebook-x86_64-history_line
3030
path: ${{ inputs.histLineDir }}
3131
- name: Download artifact 📥
3232
uses: actions/download-artifact@v3
@@ -36,7 +36,7 @@ runs:
3636
- name: Download artifact 📥
3737
uses: actions/download-artifact@v3
3838
with:
39-
name: minimal-notebook-amd64-history_line
39+
name: minimal-notebook-x86_64-history_line
4040
path: ${{ inputs.histLineDir }}
4141
- name: Download artifact 📥
4242
uses: actions/download-artifact@v3
@@ -46,7 +46,7 @@ runs:
4646
- name: Download artifact 📥
4747
uses: actions/download-artifact@v3
4848
with:
49-
name: scipy-notebook-amd64-history_line
49+
name: scipy-notebook-x86_64-history_line
5050
path: ${{ inputs.histLineDir }}
5151
- name: Download artifact 📥
5252
uses: actions/download-artifact@v3
@@ -56,12 +56,12 @@ runs:
5656
- name: Download artifact 📥
5757
uses: actions/download-artifact@v3
5858
with:
59-
name: r-notebook-amd64-history_line
59+
name: r-notebook-x86_64-history_line
6060
path: ${{ inputs.histLineDir }}
6161
- name: Download artifact 📥
6262
uses: actions/download-artifact@v3
6363
with:
64-
name: tensorflow-notebook-amd64-history_line
64+
name: tensorflow-notebook-x86_64-history_line
6565
path: ${{ inputs.histLineDir }}
6666
- name: Download artifact 📥
6767
uses: actions/download-artifact@v3
@@ -71,7 +71,7 @@ runs:
7171
- name: Download artifact 📥
7272
uses: actions/download-artifact@v3
7373
with:
74-
name: datascience-notebook-amd64-history_line
74+
name: datascience-notebook-x86_64-history_line
7575
path: ${{ inputs.histLineDir }}
7676
- name: Download artifact 📥
7777
uses: actions/download-artifact@v3
@@ -81,7 +81,7 @@ runs:
8181
- name: Download artifact 📥
8282
uses: actions/download-artifact@v3
8383
with:
84-
name: pyspark-notebook-amd64-history_line
84+
name: pyspark-notebook-x86_64-history_line
8585
path: ${{ inputs.histLineDir }}
8686
- name: Download artifact 📥
8787
uses: actions/download-artifact@v3
@@ -91,7 +91,7 @@ runs:
9191
- name: Download artifact 📥
9292
uses: actions/download-artifact@v3
9393
with:
94-
name: all-spark-notebook-amd64-history_line
94+
name: all-spark-notebook-x86_64-history_line
9595
path: ${{ inputs.histLineDir }}
9696

9797
- name: Download artifact 📥
@@ -102,7 +102,7 @@ runs:
102102
- name: Download artifact 📥
103103
uses: actions/download-artifact@v3
104104
with:
105-
name: base-notebook-amd64-manifest
105+
name: base-notebook-x86_64-manifest
106106
path: ${{ inputs.manifestDir }}
107107
- name: Download artifact 📥
108108
uses: actions/download-artifact@v3
@@ -112,7 +112,7 @@ runs:
112112
- name: Download artifact 📥
113113
uses: actions/download-artifact@v3
114114
with:
115-
name: minimal-notebook-amd64-manifest
115+
name: minimal-notebook-x86_64-manifest
116116
path: ${{ inputs.manifestDir }}
117117
- name: Download artifact 📥
118118
uses: actions/download-artifact@v3
@@ -122,7 +122,7 @@ runs:
122122
- name: Download artifact 📥
123123
uses: actions/download-artifact@v3
124124
with:
125-
name: scipy-notebook-amd64-manifest
125+
name: scipy-notebook-x86_64-manifest
126126
path: ${{ inputs.manifestDir }}
127127
- name: Download artifact 📥
128128
uses: actions/download-artifact@v3
@@ -132,12 +132,12 @@ runs:
132132
- name: Download artifact 📥
133133
uses: actions/download-artifact@v3
134134
with:
135-
name: r-notebook-amd64-manifest
135+
name: r-notebook-x86_64-manifest
136136
path: ${{ inputs.manifestDir }}
137137
- name: Download artifact 📥
138138
uses: actions/download-artifact@v3
139139
with:
140-
name: tensorflow-notebook-amd64-manifest
140+
name: tensorflow-notebook-x86_64-manifest
141141
path: ${{ inputs.manifestDir }}
142142
- name: Download artifact 📥
143143
uses: actions/download-artifact@v3
@@ -147,7 +147,7 @@ runs:
147147
- name: Download artifact 📥
148148
uses: actions/download-artifact@v3
149149
with:
150-
name: datascience-notebook-amd64-manifest
150+
name: datascience-notebook-x86_64-manifest
151151
path: ${{ inputs.manifestDir }}
152152
- name: Download artifact 📥
153153
uses: actions/download-artifact@v3
@@ -157,7 +157,7 @@ runs:
157157
- name: Download artifact 📥
158158
uses: actions/download-artifact@v3
159159
with:
160-
name: pyspark-notebook-amd64-manifest
160+
name: pyspark-notebook-x86_64-manifest
161161
path: ${{ inputs.manifestDir }}
162162
- name: Download artifact 📥
163163
uses: actions/download-artifact@v3
@@ -167,5 +167,5 @@ runs:
167167
- name: Download artifact 📥
168168
uses: actions/download-artifact@v3
169169
with:
170-
name: all-spark-notebook-amd64-manifest
170+
name: all-spark-notebook-x86_64-manifest
171171
path: ${{ inputs.manifestDir }}

‎.github/workflows/docker-build-test-upload.yml‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Download parent image, build new one, test it and upload to GitHub artifacts
1+
name: Download parent image, build new one, test and upload it and tags and manifests to GitHub artifacts
22

33
env:
44
OWNER: ${{ github.repository_owner }}
@@ -35,9 +35,13 @@ jobs:
3535
platform: ${{ inputs.platform }}
3636

3737
# Self-hosted runners share a state (whole VM) between runs
38-
- name: Reset docker state 🗑️
39-
if: ${{ inputs.platform != 'amd64' }}
40-
run: docker system prune --all --force
38+
- name: Reset docker state and cleanup artifacts 🗑️
39+
if: ${{ inputs.platform != 'x86_64' }}
40+
run: |
41+
docker system prune --all --force
42+
rm -rf /tmp/tags/
43+
rm -rf /tmp/hist_lines/
44+
rm -rf /tmp/manifests/
4145
shell: bash
4246

4347
- name: Load parent built image to Docker 📥
@@ -59,6 +63,33 @@ jobs:
5963
run: python3 -m tests.run_tests --short-image-name ${{ inputs.image }} --owner ${{ env.OWNER }}
6064
shell: bash
6165

66+
- name: Write tags file 🏷
67+
run: |
68+
python3 -m tagging.write_tags_file --short-image-name ${{ inputs.image }} --tags-dir /tmp/tags/ --owner ${{ env.OWNER }}
69+
shell: bash
70+
- name: Upload tags file 💾
71+
uses: actions/upload-artifact@v3
72+
with:
73+
name: ${{ inputs.image }}-${{ inputs.platform }}-tags
74+
path: /tmp/tags/${{ inputs.platform }}-${{ inputs.image }}.txt
75+
retention-days: 3
76+
77+
- name: Write manifest and build history file 🏷
78+
run: python3 -m tagging.write_manifest --short-image-name ${{ inputs.image }} --hist-line-dir /tmp/hist_lines/ --manifest-dir /tmp/manifests/ --owner ${{ env.OWNER }}
79+
shell: bash
80+
- name: Upload manifest file 💾
81+
uses: actions/upload-artifact@v3
82+
with:
83+
name: ${{ inputs.image }}-${{ inputs.platform }}-manifest
84+
path: /tmp/manifests/${{ inputs.platform }}-${{ inputs.image }}-*.md
85+
retention-days: 3
86+
- name: Upload build history line 💾
87+
uses: actions/upload-artifact@v3
88+
with:
89+
name: ${{ inputs.image }}-${{ inputs.platform }}-history_line
90+
path: /tmp/hist_lines/${{ inputs.platform }}-${{ inputs.image }}-*.txt
91+
retention-days: 3
92+
6293
- name: Save image as a tar for later use 💾
6394
run: docker save ${{ env.OWNER }}/${{ inputs.image }} -o /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar
6495
shell: bash

‎.github/workflows/docker-merge-tags.yml‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
name: Download Docker images from GitHub artifacts and merge tags
2+
3+
env:
4+
OWNER: ${{ github.repository_owner }}
5+
6+
on:
7+
workflow_call:
8+
inputs:
9+
images:
10+
description: Stringified JSON object listing image names
11+
required: true
12+
type: string
13+
secrets:
14+
DOCKERHUB_USERNAME:
15+
required: true
16+
DOCKERHUB_TOKEN:
17+
required: true
18+
19+
jobs:
20+
tag-push:
21+
runs-on: ubuntu-latest
22+
23+
strategy:
24+
matrix:
25+
image: ${{ fromJson(inputs.images) }}
26+
27+
steps:
28+
- name: Checkout Repo ⚡️
29+
uses: actions/checkout@v3
30+
- name: Create dev environment 📦
31+
uses: ./.github/actions/create-dev-env
32+
with:
33+
platform: x86_64
34+
35+
- name: Download x86_64 tags file 📥
36+
uses: actions/download-artifact@v3
37+
with:
38+
name: ${{ matrix.image }}-x86_64-tags
39+
path: /tmp/tags/
40+
41+
- name: Download aarch64 tags file 📥
42+
if: matrix.image != 'tensorflow-notebook'
43+
uses: actions/download-artifact@v3
44+
with:
45+
name: ${{ matrix.image }}-aarch64-tags
46+
path: /tmp/tags/
47+
48+
- name: Create an empty aarch64 tags file for tensorflow-notebook 💩
49+
if: matrix.image == 'tensorflow-notebook'
50+
run: touch /tmp/tags/aarch64-tensorflow-notebook.txt
51+
shell: bash
52+
53+
- name: Login to Docker Hub 🔐
54+
if: github.ref == 'refs/heads/main' || github.event_name == 'schedule'
55+
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # dependabot updates to latest release
56+
with:
57+
username: ${{ secrets.DOCKERHUB_USERNAME }}
58+
password: ${{ secrets.DOCKERHUB_TOKEN }}
59+
60+
- name: Merge tags for the images 🔀
61+
if: github.ref == 'refs/heads/main' || github.event_name == 'schedule'
62+
run: python3 -m tagging.merge_tags --short-image-name ${{ matrix.image }} --tags-dir /tmp/tags/
63+
shell: bash

‎.github/workflows/docker-tag-manifest-push.yml‎

Lines changed: 10 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,6 @@ on:
1414
description: Image platform
1515
required: true
1616
type: string
17-
runsOn:
18-
description: GitHub Actions Runner image
19-
required: true
20-
type: string
2117
secrets:
2218
DOCKERHUB_USERNAME:
2319
required: true
@@ -26,7 +22,7 @@ on:
2622

2723
jobs:
2824
tag-push:
29-
runs-on: ${{ inputs.runsOn }}
25+
runs-on: ubuntu-latest
3026

3127
strategy:
3228
matrix:
@@ -38,57 +34,28 @@ jobs:
3834
- name: Create dev environment 📦
3935
uses: ./.github/actions/create-dev-env
4036
with:
41-
platform: ${{ inputs.platform }}
42-
43-
# Self-hosted runners share a state (whole VM) between runs
44-
- name: Reset docker state and cleanup artifacts 🗑️
45-
if: ${{ inputs.platform != 'amd64' }}
46-
run: |
47-
docker system prune --all --force
48-
rm -rf /tmp/hist_lines/
49-
rm -rf /tmp/manifests/
50-
shell: bash
51-
37+
platform: x86_64
5238
- name: Load image to Docker 📥
5339
uses: ./.github/actions/load-image
5440
with:
5541
image: ${{ matrix.image }}
5642
platform: ${{ inputs.platform }}
5743

58-
- name: Create tags 🏷
59-
run: |
60-
python3 -m tagging.tag_image --short-image-name ${{ matrix.image }} --owner ${{ env.OWNER }}
61-
docker image ls -a
62-
shell: bash
63-
64-
- name: Write manifest and build history file 🏷
65-
run: python3 -m tagging.write_manifest --short-image-name ${{ matrix.image }} --hist-line-dir /tmp/hist_lines/ --manifest-dir /tmp/manifests/ --owner ${{ env.OWNER }}
66-
shell: bash
67-
- name: Upload manifest file 💾
68-
uses: actions/upload-artifact@v3
69-
with:
70-
name: ${{ matrix.image }}-${{ inputs.platform }}-manifest
71-
path: /tmp/manifests/${{ inputs.platform }}-${{ matrix.image }}-*.md
72-
retention-days: 3
73-
- name: Upload build history line 💾
74-
uses: actions/upload-artifact@v3
75-
with:
76-
name: ${{ matrix.image }}-${{ inputs.platform }}-history_line
77-
path: /tmp/hist_lines/${{ inputs.platform }}-${{ matrix.image }}-*.txt
78-
retention-days: 3
79-
80-
- name: Remove aarch64 latest tag 🗑️
81-
if: ${{ inputs.platform != 'amd64' }}
82-
run: docker rmi ${{ env.OWNER }}/${{ matrix.image }}
83-
shell: bash
84-
8544
- name: Login to Docker Hub 🔐
8645
if: github.ref == 'refs/heads/main' || github.event_name == 'schedule'
8746
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # dependabot updates to latest release
8847
with:
8948
username: ${{ secrets.DOCKERHUB_USERNAME }}
9049
password: ${{ secrets.DOCKERHUB_TOKEN }}
9150

51+
- name: Download tags file 📥
52+
uses: actions/download-artifact@v3
53+
with:
54+
name: ${{ matrix.image }}-${{ inputs.platform }}-tags
55+
path: /tmp/tags/
56+
- name: Apply tags to the loaded image 🏷
57+
run: python3 -m tagging.apply_tags --short-image-name ${{ matrix.image }} --tags-dir /tmp/tags/ --platform ${{ inputs.platform }} --owner ${{ env.OWNER }}
58+
9259
- name: Push Images to Docker Hub 📤
9360
if: github.ref == 'refs/heads/main' || github.event_name == 'schedule'
9461
run: docker push --all-tags ${{ env.OWNER }}/${{ matrix.image }}

‎.github/workflows/docker-wiki-update.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ jobs:
1515
- name: Create dev environment 📦
1616
uses: ./.github/actions/create-dev-env
1717
with:
18-
platform: amd64
18+
platform: x86_64
1919

2020
- name: Download all manifests and history lines 📥
2121
uses: ./.github/actions/download-manifests

0 commit comments

Comments
 (0)