Skip to content

Commit 1ef03fa

Browse files
pelsonpelson
committed
Merge remote-tracking branch 'upstream/v2.0.x'
* upstream/v2.0.x: Improved the way requirements are handled, ensuring that pip gets all the information it needs. (#2890)
2 parents 72025c1 + e674846 commit 1ef03fa

File tree

11 files changed

+237
-115
lines changed

11 files changed

+237
-115
lines changed

‎.travis.yml

Lines changed: 60 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -24,80 +24,97 @@ env:
2424
- secure: "N9/qBUT5CqfC7KQBDy5mIWZcGNuUJk3e/qmKJpotWYV+zwOI4GghJsRce6nFnlRiwl65l5oBEcvf3+sBvUfbZqh7U0MdHpw2tHhr2FSCmMB3bkvARZblh9M37f4da9G9VmRkqnyBM5G5TImXtoq4dusvNWKvLW0qETciaipq7ws="
2525

2626
git:
27+
# We need a deep clone so that we can compute the age of the files using their git history.
2728
depth: 10000
2829

2930
install:
30-
- export IRIS_TEST_DATA_REF="2f3a6bcf25f81bd152b3d66223394074c9069a96"
31-
- export IRIS_TEST_DATA_SUFFIX=$(echo "${IRIS_TEST_DATA_REF}" | sed "s/^v//")
31+
- >
32+
export IRIS_TEST_DATA_REF="2f3a6bcf25f81bd152b3d66223394074c9069a96";
33+
export IRIS_TEST_DATA_SUFFIX=$(echo "${IRIS_TEST_DATA_REF}" | sed "s/^v//");
3234
3335
# Install miniconda
3436
# -----------------
35-
- export CONDA_BASE=https://repo.continuum.io/miniconda/Miniconda
36-
- if [[ "$TRAVIS_PYTHON_VERSION" == 2* ]]; then
37-
wget ${CONDA_BASE}2-latest-Linux-x86_64.sh -O miniconda.sh;
37+
- >
38+
echo 'Installing miniconda';
39+
export CONDA_BASE=https://repo.continuum.io/miniconda/Miniconda;
40+
if [[ "$TRAVIS_PYTHON_VERSION" == 2* ]]; then
41+
wget --quiet ${CONDA_BASE}2-latest-Linux-x86_64.sh -O miniconda.sh;
3842
else
39-
wget ${CONDA_BASE}3-latest-Linux-x86_64.sh -O miniconda.sh;
40-
fi
41-
- bash miniconda.sh -b -p $HOME/miniconda
42-
- export PATH="$HOME/miniconda/bin:$PATH"
43+
wget --quiet ${CONDA_BASE}3-latest-Linux-x86_64.sh -O miniconda.sh;
44+
fi;
45+
bash miniconda.sh -b -p $HOME/miniconda;
46+
export PATH="$HOME/miniconda/bin:$PATH";
4347
4448
# Create the basic testing environment
4549
# ------------------------------------
4650
# Explicitly add defaults channel, see https://github.com/conda/conda/issues/2675
47-
- conda config --add channels defaults
48-
- conda config --set always_yes yes --set changeps1 no
49-
- conda config --set show_channel_urls True
50-
- conda update --quiet conda
51-
- ENV_NAME='test-environment'
52-
- conda create --quiet -n $ENV_NAME python=$TRAVIS_PYTHON_VERSION
53-
- source activate $ENV_NAME
51+
- >
52+
echo 'Configure conda and create an environment';
53+
conda config --set always_yes yes --set changeps1 no;
54+
conda config --set show_channel_urls True;
55+
conda config --add channels conda-forge;
56+
conda update --quiet conda;
57+
ENV_NAME='test-environment';
58+
conda create --quiet -n $ENV_NAME python=$TRAVIS_PYTHON_VERSION pip;
59+
source activate $ENV_NAME;
5460
5561
# Customise the testing environment
5662
# ---------------------------------
57-
- conda config --add channels conda-forge
58-
- if [[ "$TEST_MINIMAL" == true ]]; then
59-
conda install --quiet --file minimal-conda-requirements.txt;
60-
else
61-
if [[ "$TRAVIS_PYTHON_VERSION" == 3* ]]; then
62-
sed -e '/esmpy/d' -e 's/#.\+$//' conda-requirements.txt | xargs conda install --quiet;
63-
else
64-
conda install --quiet --file conda-requirements.txt;
65-
fi
66-
fi
67-
68-
# JUST FOR NOW : Install latest version of iris-grib.
69-
# TODO : remove when this release is available on conda-forge.
70-
- if [[ "$TEST_MINIMAL" != true ]]; then
71-
pip install git+https://github.com/SciTools/[email protected] ;
72-
fi
63+
- >
64+
echo 'Install Iris dependencies';
65+
CONDA_REQS_FLAGS="";
66+
CONDA_REQS_GROUPS="test";
67+
if [[ "$TRAVIS_PYTHON_VERSION" == 2* ]]; then
68+
CONDA_REQS_FLAGS="${CONDA_REQS_FLAGS} --py2";
69+
fi;
70+
if [[ "$TEST_MINIMAL" != true ]]; then
71+
CONDA_REQS_GROUPS="${CONDA_REQS_GROUPS} all";
72+
fi;
73+
if [[ "${TEST_TARGET}" == 'doctest' ]]; then
74+
CONDA_REQS_GROUPS="${CONDA_REQS_GROUPS} docs";
75+
fi;
76+
CONDA_REQS_FILE=conda-requirements.txt;
77+
python requirements/gen_conda_requirements.py ${CONDA_REQS_FLAGS} --groups ${CONDA_REQS_GROUPS} > ${CONDA_REQS_FILE};
78+
cat ${CONDA_REQS_FILE};
79+
conda install --quiet --file ${CONDA_REQS_FILE};
7380
7481
- PREFIX=$HOME/miniconda/envs/$ENV_NAME
7582

7683
# Output debug info
77-
- conda list
78-
- conda info -a
84+
- >
85+
conda list;
86+
conda info -a;
7987
8088
# Pre-load Natural Earth data to avoid multiple, overlapping downloads.
8189
# i.e. There should be no DownloadWarning reports in the log.
8290
- python -c 'import cartopy; cartopy.io.shapereader.natural_earth()'
8391

8492
# iris test data
85-
- if [[ "$TEST_MINIMAL" != true ]]; then
86-
wget -O iris-test-data.zip https://github.com/SciTools/iris-test-data/archive/${IRIS_TEST_DATA_REF}.zip;
93+
- >
94+
if [[ "$TEST_MINIMAL" != true ]]; then
95+
wget --quiet -O iris-test-data.zip https://github.com/SciTools/iris-test-data/archive/${IRIS_TEST_DATA_REF}.zip;
8796
unzip -q iris-test-data.zip;
88-
ln -s $(pwd)/iris-test-data-${IRIS_TEST_DATA_SUFFIX} iris-test-data;
97+
mv "iris-test-data-${IRIS_TEST_DATA_SUFFIX}" iris-test-data;
8998
fi
9099
91100
# set config paths
92-
- SITE_CFG=lib/iris/etc/site.cfg
93-
- echo "[Resources]" > $SITE_CFG
94-
- echo "test_data_dir = $(pwd)/iris-test-data/test_data" >> $SITE_CFG
95-
- echo "doc_dir = $(pwd)/docs/iris" >> $SITE_CFG
96-
- echo "[System]" >> $SITE_CFG
97-
- echo "udunits2_path = $PREFIX/lib/libudunits2.so" >> $SITE_CFG
101+
- >
102+
SITE_CFG=lib/iris/etc/site.cfg;
103+
echo "[Resources]" > $SITE_CFG;
104+
echo "test_data_dir = $(pwd)/iris-test-data/test_data" >> $SITE_CFG;
105+
echo "doc_dir = $(pwd)/docs/iris" >> $SITE_CFG;
106+
echo "[System]" >> $SITE_CFG;
107+
echo "udunits2_path = $PREFIX/lib/libudunits2.so" >> $SITE_CFG;
98108
99109
- python setup.py --quiet install
100110

111+
# JUST FOR NOW : Install latest version of iris-grib.
112+
# TODO : remove when iris doesn't do an integration test requiring iris-grib.
113+
- if [[ "$TEST_MINIMAL" != true && ${TRAVIS_PYTHON_VERSION} == 2* ]]; then
114+
conda install python-ecmwf_grib;
115+
pip install git+https://github.com/SciTools/[email protected];
116+
fi
117+
101118
script:
102119
- if [[ $TEST_TARGET == 'default' ]]; then
103120
python -m iris.tests.runner --default-tests --system-tests --print-failed-images --num-processors=3;

‎conda-requirements.txt

Lines changed: 0 additions & 38 deletions
This file was deleted.

‎minimal-conda-requirements.txt

Lines changed: 0 additions & 27 deletions
This file was deleted.

‎requirements/all.txt

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Dependencies for a feature complete installation
2+
# ------------------------------------------------
3+
4+
# esmpy regridding not available through pip.
5+
#conda: esmpy>=7.0 (only python=2)
6+
gdal
7+
mo_pack
8+
nc_time_axis
9+
pandas
10+
stratify #conda: python-stratify
11+
pyugrid
12+
13+
#conda: graphviz
14+
15+
# Iris sample data is not available through pip. It can be installed from
16+
# https://github.com/SciTools/iris-sample-data/archive/master.zip
17+
#conda: iris-sample-data

‎requirements/core.txt

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# Absolute minimal dependencies for iris
2+
# --------------------------------------
3+
4+
# Without these, iris won't even import.
5+
6+
cartopy
7+
matplotlib<1.9
8+
netcdf4
9+
numpy
10+
# pyke (not pip installable) #conda: pyke
11+
cf_units
12+
dask>=0.15.0

‎requirements/docs.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
sphinx

‎requirements/extensions.txt

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# Iris extensions (i.e. key tools that depend on Iris)
2+
# ----------------------------------------------------
3+
4+
# Note: pip can handle the circularity of these extensions, but conda will
5+
# struggle. To install these extensions, ensure iris[core] has been installed
6+
# first.
7+
8+
iris_grib;python_version<"3" #conda:

‎requirements/gen_conda_requirements.py

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
# (C) British Crown Copyright 2017, Met Office
2+
#
3+
# This file is part of Iris.
4+
#
5+
# Iris is free software: you can redistribute it and/or modify it under
6+
# the terms of the GNU Lesser General Public License as published by the
7+
# Free Software Foundation, either version 3 of the License, or
8+
# (at your option) any later version.
9+
#
10+
# Iris is distributed in the hope that it will be useful,
11+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
# GNU Lesser General Public License for more details.
14+
#
15+
# You should have received a copy of the GNU Lesser General Public License
16+
# along with Iris. If not, see <http://www.gnu.org/licenses/>.
17+
18+
import argparse
19+
import os.path
20+
21+
22+
REQS_DIR = os.path.dirname(__file__)
23+
CONDA_PATTERN = '#conda:'
24+
25+
26+
def read_conda_reqs(fname, options):
27+
lines = []
28+
with open(fname, 'r') as fh:
29+
for line in fh:
30+
line = line.strip()
31+
if '#conda:' in line:
32+
line_start = line.index(CONDA_PATTERN) + len(CONDA_PATTERN)
33+
line = line[line_start:].strip()
34+
if 'only python=2' in line:
35+
if 'python=2' in options:
36+
line = line.replace('(only python=2)', '')
37+
lines.append(line)
38+
else:
39+
continue
40+
else:
41+
lines.append(line)
42+
else:
43+
lines.append(line)
44+
return lines
45+
46+
47+
def compute_requirements(requirement_names=('core', ), options=None):
48+
conda_reqs_lines = []
49+
50+
if 'python=2' in options:
51+
conda_reqs_lines.append('python=2.*')
52+
else:
53+
conda_reqs_lines.append('# Python 3 conda configuration')
54+
55+
for req_name in requirement_names:
56+
fname = os.path.join(REQS_DIR, '{}.txt'.format(req_name))
57+
if not os.path.exists(fname):
58+
raise RuntimeError('Unable to find the requirements file for {} '
59+
'in {}'.format(req_name, fname))
60+
conda_reqs_lines.extend(read_conda_reqs(fname, options))
61+
conda_reqs_lines.append('')
62+
63+
return conda_reqs_lines
64+
65+
66+
def main():
67+
parser = argparse.ArgumentParser()
68+
parser.add_argument("--verbosity", help="increase output verbosity")
69+
parser.add_argument(
70+
"--groups", nargs='*', default=[],
71+
help=("Gather requirements for these given named groups "
72+
"(as found in the requirements/ folder)"))
73+
parser.add_argument(
74+
"--py2", action="store_true",
75+
help="Build the conda requirements for a python 2 installation")
76+
77+
args = parser.parse_args()
78+
79+
requirement_names = args.groups
80+
requirement_names.insert(0, 'core')
81+
requirement_names.insert(0, 'setup')
82+
83+
options = []
84+
if args.py2:
85+
options.append('python=2')
86+
87+
print('\n'.join(compute_requirements(requirement_names, options)))
88+
89+
90+
if __name__ == '__main__':
91+
main()

‎requirements/setup.txt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Dependencies necessary to run setup.py of iris
2+
# ----------------------------------------------
3+
4+
setuptools
5+
# pyke (not pip installable) #conda: pyke
6+
six

‎requirements/test.txt

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Dependencies needed to run the iris tests
2+
#------------------------------------------
3+
4+
mock
5+
nose
6+
pep8
7+
filelock
8+
imagehash
9+
requests

0 commit comments

Comments
 (0)