Skip to content

Commit 41bdac5

Browse files
wz1000wz1000
committed
Add release scripts
1 parent 5ce21d0 commit 41bdac5

File tree

3 files changed

+237
-0
lines changed

3 files changed

+237
-0
lines changed

‎release/README.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Making and uploading the Gitlab release to downloads.haskell.org
2+
3+
1. Run the gitlab release pipeline using https://gitlab.haskell.org/haskell/haskell-language-server/-/pipelines/new
4+
2. Once the pipeline has completed, download the artifacts using `fetch_gitlab.py`
5+
- For example for the `1.7.0.0` release: `python fetch_gitlab.py -p <pipeline_id> --output haskell-language-server-1.7.0.0 -r 1.7.0.0`
6+
- Ensure all the artifacts in the output directory are accurate and add any missing/extra artifacts
7+
3. `cd` to the output directory created in the previous step, and run `SIGNING_KEY=<your signing key> ../upload.sh`

‎release/fetch_gitlab.py

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
# adapted from https://gitlab.haskell.org/bgamari/ghc-utils/-/blob/master/rel-eng/fetch-gitlab-artifacts/fetch_gitlab.py
2+
import logging
3+
from pathlib import Path
4+
import subprocess
5+
import gitlab
6+
7+
logging.basicConfig(level=logging.INFO)
8+
9+
def strip_prefix(s, prefix):
10+
if s.startswith(prefix):
11+
return s[len(prefix):]
12+
else:
13+
return None
14+
15+
def fetch_artifacts(release: str, pipeline_id: int,
16+
dest_dir: Path, gl: gitlab.Gitlab):
17+
dest_dir.mkdir(exist_ok=True)
18+
proj = gl.projects.get('haskell/haskell-language-server')
19+
pipeline = proj.pipelines.get(pipeline_id)
20+
tmpdir = Path("fetch-gitlab")
21+
tmpdir.mkdir(exist_ok=True)
22+
for pipeline_job in pipeline.jobs.list(all=True):
23+
if len(pipeline_job.artifacts) == 0:
24+
logging.info(f'job {pipeline_job.name} ({pipeline_job.id}) has no artifacts')
25+
continue
26+
27+
job = proj.jobs.get(pipeline_job.id)
28+
platform = strip_prefix(job.name, 'tar-')
29+
if not platform:
30+
logging.info(f'Skipping {job.name} (not a tar job)')
31+
continue
32+
try:
33+
destdir = tmpdir / job.name
34+
zip_name = Path(f"{tmpdir}/{job.name}.zip")
35+
if not zip_name.exists() or zip_name.stat().st_size == 0:
36+
logging.info(f'downloading archive {zip_name} for job {job.name} (job {job.id})...')
37+
with open(zip_name, 'wb') as f:
38+
job.artifacts(streamed=True, action=f.write)
39+
40+
if zip_name.stat().st_size == 0:
41+
logging.info(f'artifact archive for job {job.name} (job {job.id}) is empty')
42+
continue
43+
44+
dest = dest_dir / f'haskell-language-server-{release}-{platform}.tar.xz'
45+
if dest.exists():
46+
logging.info(f'bindist {dest} already exists')
47+
continue
48+
49+
subprocess.run(['unzip', '-bo', zip_name, '-d', destdir])
50+
bindist_files = list(destdir.glob('*/haskell-language-server*.tar.xz'))
51+
if len(bindist_files) == 0:
52+
logging.warn(f'Bindist does not exist')
53+
continue
54+
55+
bindist = bindist_files[0]
56+
logging.info(f'extracted {job.name} to {dest}')
57+
bindist.replace(dest)
58+
except Exception as e:
59+
logging.error(f'Error fetching job {job.name}: {e}')
60+
pass
61+
62+
def main():
63+
import argparse
64+
parser = argparse.ArgumentParser()
65+
parser.add_argument('--pipeline', '-p', required=True, type=int, help="pipeline id")
66+
parser.add_argument('--release', '-r', required=True, type=str, help="release name")
67+
parser.add_argument('--output', '-o', type=Path, default=Path.cwd(), help="output directory")
68+
parser.add_argument('--profile', '-P', default='haskell',
69+
help='python-gitlab.cfg profile name')
70+
args = parser.parse_args()
71+
gl = gitlab.Gitlab.from_config(args.profile)
72+
fetch_artifacts(args.release, args.pipeline,
73+
dest_dir=args.output, gl=gl)
74+
75+
if __name__ == '__main__':
76+
main()

‎release/upload.sh

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
# This is a script for preparing and uploading a release of Haskell Language Server.
6+
# Adapted from https://gitlab.haskell.org/bgamari/ghc-utils/-/commits/master/rel-eng/upload.sh
7+
#
8+
# Usage,
9+
# 1. Set $SIGNING_KEY to your key id (prefixed with '=')
10+
# 2. Create a directory called haskell-langauge-server-<release_number> and place the binary tarballs there
11+
# 4. Run this script from that directory
12+
#
13+
# You can also invoke the script with an argument to perform only
14+
# a subset of the usual release,
15+
#
16+
# upload.sh gen_hashes generate signed hashes of the release
17+
# tarballs
18+
# upload.sh sign generate signed hashes of the release
19+
# tarballs
20+
# upload.sh upload upload the tarballs and documentation
21+
# to downloads.haskell.org
22+
#
23+
# Prerequisites: moreutils
24+
25+
# Infer release name from directory name
26+
if [ -z "$rel_name" ]; then
27+
rel_name="$(basename $(pwd))"
28+
fi
29+
30+
# Infer version from tarball names
31+
if [ -z "$ver" ]; then
32+
ver="$(ls haskell-language-server-*.tar.* | sed -ne 's/haskell-language-server-\([0-9]\+\.[0-9]\+\.[0-9]\+\(\.[0-9]\+\)\?\).\+/\1/p' | head -n1)"
33+
if [ -z "$ver" ]; then echo "Failed to infer \$ver"; exit 1; fi
34+
fi
35+
36+
echo HLS version $ver
37+
38+
host="webhost.haskell.org"
39+
40+
usage() {
41+
echo "Usage: [rel_name=<name>] SIGNING_KEY=<key> $0 <action>"
42+
echo
43+
echo "where,"
44+
echo " rel_name gives the release name (e.g. 1.7.0.0)"
45+
echo "and <action> is one of,"
46+
echo " [nothing] do everything below"
47+
echo " gen_hashes generated hashes of the release tarballs"
48+
echo " sign sign hashes of the release tarballs"
49+
echo " upload upload the tarballs and documentation to downloads.haskell.org"
50+
echo " purge_all purge entire release from the CDN"
51+
echo " purge_file file purge a given file from the CDN"
52+
echo " verify verify the signatures in this directory"
53+
echo
54+
}
55+
56+
if [ -z "$ver" ]; then
57+
usage
58+
exit 1
59+
fi
60+
if [ -z "$rel_name" ]; then
61+
rel_name="$ver"
62+
fi
63+
64+
# returns the set of files that must have hashes generated.
65+
function hash_files() {
66+
echo $(find -maxdepth 1 \
67+
-iname '*.xz' \
68+
-o -iname '*.lz' \
69+
-o -iname '*.bz2' \
70+
-o -iname '*.zip' \
71+
)
72+
echo $(find -maxdepth 1 -iname '*.patch')
73+
}
74+
75+
function gen_hashes() {
76+
echo -n "Hashing..."
77+
sha1sum $(hash_files) >| SHA1SUMS &
78+
sha256sum $(hash_files) >| SHA256SUMS &
79+
wait
80+
echo "done"
81+
}
82+
83+
function sign() {
84+
# Kill DISPLAY lest pinentry won't work
85+
DISPLAY=
86+
eval "$(gpg-agent --daemon --sh --pinentry-program $(which pinentry))"
87+
for i in $(hash_files) SHA1SUMS SHA256SUMS; do
88+
if [ -e $i -a -e $i.sig -a $i.sig -nt $i ]; then
89+
echo "Skipping signing of $i"
90+
continue
91+
elif [ -e $i.sig ] && gpg2 --verify $i.sig; then
92+
# Don't resign if current signature is valid
93+
touch $i.sig
94+
continue
95+
fi
96+
echo "Signing $i"
97+
rm -f $i.sig
98+
gpg2 --use-agent --detach-sign --local-user="$SIGNING_KEY" $i
99+
done
100+
}
101+
102+
function verify() {
103+
if [ $(find -iname '*.sig' | wc -l) -eq 0 ]; then
104+
echo "No signatures to verify"
105+
return
106+
fi
107+
108+
for i in *.sig; do
109+
echo
110+
echo Verifying $i
111+
gpg2 --verify $i $(basename $i .sig)
112+
done
113+
}
114+
115+
function upload() {
116+
verify
117+
chmod ugo+r,o-w -R .
118+
dir=$(echo $rel_name | sed s/-release//)
119+
lftp -c " \
120+
open -u hls: sftp://$host && \
121+
mirror -P20 -c --reverse --exclude=fetch-gitlab --exclude=out . hls/$dir && \
122+
wait all;"
123+
chmod ugo-w $(ls *.xz *.bz2 *.zip)
124+
}
125+
126+
function purge_all() {
127+
# Purge CDN cache
128+
curl -X PURGE http://downloads.haskell.org/hls/
129+
curl -X PURGE http://downloads.haskell.org/~hls/
130+
curl -X PURGE http://downloads.haskell.org/hls/$dir
131+
curl -X PURGE http://downloads.haskell.org/hls/$dir/
132+
curl -X PURGE http://downloads.haskell.org/~hls/$dir
133+
curl -X PURGE http://downloads.haskell.org/~hls/$dir/
134+
for i in *; do
135+
purge_file $i
136+
done
137+
}
138+
139+
function purge_file() {
140+
curl -X PURGE http://downloads.haskell.org/~hls/$rel_name/$i
141+
curl -X PURGE http://downloads.haskell.org/~hls/$rel_name/$i/
142+
curl -X PURGE http://downloads.haskell.org/hls/$rel_name/$i
143+
curl -X PURGE http://downloads.haskell.org/hls/$rel_name/$i/
144+
}
145+
146+
147+
if [ "x$1" == "x" ]; then
148+
gen_hashes
149+
sign
150+
upload
151+
purge_all
152+
else
153+
$@
154+
fi

0 commit comments

Comments
 (0)