Merge pull request pytorch#146 from pjh5/always_upload

Allowing individual uploads of builds

Author: pjh5

conda/build_pytorch.sh

@@ -10,24 +10,17 @@ set -ex
 # There's also duplicate versioning logic amongst *all* the building scripts
 
 # Env variables that should be set
-# LINUX env variables that should be set
-#   HOST_PACKAGE_DIR
-#     Absolute path (in docker space) to folder where final packages will be
-#     stored.
+# PYTORCH_FINAL_PACKAGE_DIR
+#   Absolute path (in docker space) to folder where final packages will be
+#   stored.
 #
 # MACOS env variables that should be set
-#   MAC_CONDA_FINAL_FOLDER
-#     **Absolute** path to folder where final packages will be stored.
-#
 #   MAC_PACKAGE_WORK_DIR
 #     Absolute path to a workdir in which to clone an isolated conda
 #     installation and pytorch checkout. If the pytorch checkout already exists
 #     then it will not be overwritten.
 #
 # WINDOWS env variables that should be set
-#   WIN_CONDA_FINAL_FOLDER
-#     **Absolute** path to folder where final packages will be stored.
-#
 #   WIN_PACKAGE_WORK_DIR
 #     Absolute path to a workdir in which to clone an isolated conda
 #     installation and pytorch checkout. If the pytorch checkout already exists
@@ -267,17 +260,8 @@ for py_ver in "${DESIRED_PYTHON[@]}"; do
     built_package="$(find $output_folder/ -name '*pytorch*.tar.bz2')"
 
     # Copy the built package to the host machine for persistence before testing
-    if [[ -n "$HOST_PACKAGE_DIR" ]]; then
-        mkdir -p "$HOST_PACKAGE_DIR/conda_pkgs" || true
-        cp "$built_package" "$HOST_PACKAGE_DIR/conda_pkgs/"
-    fi
-    if [[ -n "$MAC_CONDA_FINAL_FOLDER" ]]; then
-        mkdir -p "$MAC_CONDA_FINAL_FOLDER" || true
-        cp "$built_package" "$MAC_CONDA_FINAL_FOLDER/"
-    fi
-    if [[ -n "$WIN_CONDA_FINAL_FOLDER" ]]; then
-        mkdir -p "$WIN_CONDA_FINAL_FOLDER" || true
-        cp "$built_package" "$WIN_CONDA_FINAL_FOLDER/"
+    if [[ -n "$PYTORCH_FINAL_PACKAGE_DIR" && -d "$PYTORCH_FINAL_PACKAGE_DIR" ]]; then
+        cp "$built_package" "$PYTORCH_FINAL_PACKAGE_DIR/"
     fi
 
     conda install -y "$built_package"

cron/build_docker.sh

@@ -7,36 +7,9 @@ SOURCE_DIR=$(cd $(dirname $0) && pwd)
 source "${SOURCE_DIR}/nightly_defaults.sh"
 
 # Handles building for manywheels and linux conda packages.
-# Env variables that should be set:
-#   PYTORCH_BUILD_VERSION
-#     This is the version string, e.g. 0.4.1 , that will be used as the
-#     pip/conda version, OR the word 'nightly', which signals all the
-#     downstream scripts to use the current date as the version number (plus
-#     other changes). This is NOT the conda build string.
-#
-#   PYTORCH_BUILD_NUMBER
-#     This is usually the number 1. If more than one build is uploaded for the
-#     same version/date, then this can be incremented to 2,3 etc in which case
-#     '.post2' will be appended to the version string of the package. This can
-#     be set to '0' only if OVERRIDE_PACKAGE_VERSION is being used to bypass
-#     all the version string logic in downstream scripts.
-#
-#   NIGHTLIES_FOLDER
-#     An arbitrary root folder to store all nightlies folders, each of which is
-#     a parent level date folder with separate subdirs for logs, wheels, conda
-#     packages, etc. This should be kept the same across all scripts called in
-#     a cron job, so it only has a default value in the top-most script
-#     build_cron.sh to avoid the default values from diverging.
-#
-#   NIGHTLIES_DATE
-#     The date in YYYY_mm_dd format that we are building for. This defaults to
-#     the current date. Sometimes cron uses a different time than that returned
-#     by `date`, so ideally this is set once by the top-most script
-#     build_cron.sh so that all scripts use the same date.
 
 # Parameters
 ##############################################################################
-
 if [[ "$#" != 3 ]]; then
   if [[ -z "$DESIRED_PYTHON" || -z "$DESIRED_CUDA" || -z "$PACKAGE_TYPE" ]]; then
       echo "The env variabled PACKAGE_TYPE must be set to 'conda' or 'manywheel' or 'libtorch'"
@@ -60,10 +33,15 @@ fi
 echo "Building a $package_type package for python$desired_python and $desired_cuda"
 echo "Starting to run the build at $(date)"
 
+# Organize folders by os, package_type, and cuda version
+host_package_dir="$(nightlies_package_folder $package_type $desired_cuda)"
+mkdir -p "$host_package_dir" || true
+
+# This is the name of the package dir dockerside
+docker_package_dir="/host_machine_pkgs"
+
 # Move to today's workspace folder
 mkdir -p "$today" || true
-host_package_dir="$today"
-docker_package_dir="/host_machine_pkgs"
 
 # Map cuda/python/storage dirs for conda or manywheel
 python_nodot="${desired_python:0:1}${desired_python:2:1}"
@@ -164,7 +142,7 @@ nvidia-docker cp "$NIGHTLIES_PYTORCH_ROOT" "$id:/pytorch"
     echo "export DESIRED_PYTHON=${desired_python}"
     echo "export DESIRED_CUDA=${desired_cuda}"
     # the following line is true from the docker's perspective
-    echo "export HOST_PACKAGE_DIR=${docker_package_dir}"
+    echo "export PYTORCH_FINAL_PACKAGE_DIR=${docker_package_dir}"
     echo "export CMAKE_ARGS=${CMAKE_ARGS[@]}"
     echo "export EXTRA_CAFFE2_CMAKE_FLAGS=${EXTRA_CAFFE2_CMAKE_FLAGS[@]}"
     echo "export RUN_TEST_PARAMS=${RUN_TEST_PARAMS}"

cron/build_mac.sh

@@ -7,35 +7,9 @@ source "${SOURCE_DIR}/nightly_defaults.sh"
 
 # Handles building for mac wheels and mac conda packages and mac libtorch packages.
 # Env variables that should be set:
-#   PYTORCH_BUILD_VERSION
-#     This is the version string, e.g. 0.4.1 , that will be used as the
-#     pip/conda version, OR the word 'nightly', which signals all the
-#     downstream scripts to use the current date as the version number (plus
-#     other changes). This is NOT the conda build string.
-#
-#   PYTORCH_BUILD_NUMBER
-#     This is usually the number 1. If more than one build is uploaded for the
-#     same version/date, then this can be incremented to 2,3 etc in which case
-#     '.post2' will be appended to the version string of the package. This can
-#     be set to '0' only if OVERRIDE_PACKAGE_VERSION is being used to bypass
-#     all the version string logic in downstream scripts.
-#
-#   NIGHTLIES_FOLDER
-#     An arbitrary root folder to store all nightlies folders, each of which is
-#     a parent level date folder with separate subdirs for logs, wheels, conda
-#     packages, etc. This should be kept the same across all scripts called in
-#     a cron job, so it only has a default value in the top-most script
-#     build_cron.sh to avoid the default values from diverging.
-#
-#   NIGHTLIES_DATE
-#     The date in YYYY_mm_dd format that we are building for. This defaults to
-#     the current date. Sometimes cron uses a different time than that returned
-#     by `date`, so ideally this is set once by the top-most script
-#     build_cron.sh so that all scripts use the same date.
 
 # Parameters
 ##############################################################################
-
 if [[ "$#" != 2 ]]; then
   if [[ -z "$DESIRED_PYTHON" || -z "$PACKAGE_TYPE" ]]; then
       echo "The env variabled PACKAGE_TYPE must be set to 'conda' or 'manywheel' or 'libtorch'"
@@ -47,20 +21,17 @@ if [[ "$#" != 2 ]]; then
 else
   package_type="$1"
   desired_python="$2"
-  desired_cuda="$3"
 fi
 if [[ "$package_type" != 'conda' && "$package_type" != 'wheel' && "$package_type" != 'libtorch' ]]; then
     echo "The package type must be 'conda' or 'wheel' or 'libtorch'"
     exit 1
 fi
+echo "$(date) Building a $package_type MacOS package for python$desired_python"
+export PYTORCH_FINAL_PACKAGE_DIR="$(nightlies_package_folder $package_type cpu)"
+mkdir -p "$PYTORCH_FINAL_PACKAGE_DIR"
 
-echo "Building a $package_type package for python$desired_python"
-echo "Starting to run the build at $(date)"
-
-# Move to today's workspace folder
-mkdir -p "$today" || true
-
-# Make the workdir for the mac builds
+# Setup a workdir
+##############################################################################
 workdir="${today}/wheel_build_dirs/${package_type}_${desired_python}"
 export MAC_PACKAGE_WORK_DIR="$workdir"
 rm -rf "$workdir"
@@ -70,21 +41,16 @@ mkdir -p "$workdir"
 cp -R "$NIGHTLIES_PYTORCH_ROOT" "$workdir"
 
 # Build the package
+##############################################################################
 if [[ "$package_type" == 'conda' ]]; then
-    # Conda package settings
     export TORCH_PACKAGE_NAME="$(echo $TORCH_PACKAGE_NAME | tr '_' '-')"
-    export MAC_PACKAGE_FINAL_FOLDER="$MAC_CONDA_FINAL_FOLDER"
     "${NIGHTLIES_BUILDER_ROOT}/conda/build_pytorch.sh"
     ret="$?"
 else
     if [[ "$package_type" == 'libtorch' ]]; then
         export BUILD_PYTHONLESS=1
     fi
-
-    # Wheel settings
     export TORCH_PACKAGE_NAME="$(echo $TORCH_PACKAGE_NAME | tr '-' '_')"
-    export MAC_PACKAGE_FINAL_FOLDER="$MAC_WHEEL_FINAL_FOLDER"
-    building_wheels=1
     "${NIGHTLIES_BUILDER_ROOT}/wheel/build_wheel.sh"
     ret="$?"
 fi

cron/build_multiple.sh

@@ -24,7 +24,7 @@ source "${SOURCE_DIR}/nightly_defaults.sh"
 if [ "$#" -lt 3 ]; then
     echo 'Illegal number of parameters'
     echo '     build_multiple.sh [conda|manywheel|wheel] DESIRED_PYTHON,s DESIRED_CUDA,s'
-    echo 'e.g. build_multiple.sh manywheel 2.7mu,3.5,3.6 cpu,cu80'
+    echo 'e.g. build_multiple.sh manywheel 2.7mu,3.5m,3.6m cpu,cu80'
     echo 'e.g. build_multiple.sh conda,manywheel 2.7 all'
     echo ' DESIRED_PYTHONs must match:   \d.\d(mu?)?'
     echo ' DESIRED_CUDAs must match  :   (cpu|cu\d\d)'
@@ -73,12 +73,9 @@ while [[ $# -gt 0 ]]; do
 done
 
 
-# Allow 'all' to translate to all python/cuda versions
-
 set +x
 echo
-echo "Starting PyTorch binary build for $NIGHTLIES_DATE"
-echo "$(date) :: Starting PyTorch binaries build"
+echo "$(date) :: Starting PyTorch binary build for $NIGHTLIES_DATE"
 echo "Building all of [${all_configs[@]}]"
 set -x
 

cron/nightly_defaults.sh

@@ -60,6 +60,7 @@ fi
 # Used in lots of places as the root dir to store all conda/wheel/manywheel
 # packages as well as logs for the day
 export today="$NIGHTLIES_FOLDER/$NIGHTLIES_DATE"
+mkdir -p "$today" || true
 
 # N.B. BUILDER_REPO and BUILDER_BRANCH are both set in cron_start.sh, as that
 # is the script that actually clones the builder repo that /this/ script is
@@ -154,12 +155,15 @@ if [[ -z "$PIP_UPLOAD_FOLDER" ]]; then
     export PIP_UPLOAD_FOLDER='nightly/'
 fi
 
-# MAC_(CONDA|WHEEL|LIBTORCH)_FINAL_FOLDER
-#   Absolute path to the folders where final conda/wheel packages should be
-#   stored
-export MAC_CONDA_FINAL_FOLDER="${today}/mac_conda_pkgs"
-export MAC_WHEEL_FINAL_FOLDER="${today}/mac_wheels"
-export MAC_LIBTORCH_FINAL_FOLDER="${today}/mac_libtorch_packages"
+# nightlies_package_folder()
+#   Function from (package_type, CUDA/cpu ver) -> where packages should be
+#   stored. OS does not factor since the folders will be on different machines
+#   and won't overlap
+#   N.B. PYTORCH_FINAL_PACKAGE_DIR is not a constant, and is not set here. This
+#   should be set by build_docker or build_mac according to this function.
+nightlies_package_folder () {
+    echo "${today}/$1/$2/"
+}
 
 # (FAILED|SUCCEEDED)_LOG_DIR
 #   Absolute path to folders that store final logs. Initially these folders

cron/upload.sh

@@ -5,27 +5,36 @@ echo "upload.sh at $(pwd) starting at $(date) on $(uname -a) with pid $$"
 SOURCE_DIR=$(cd $(dirname $0) && pwd)
 source "${SOURCE_DIR}/nightly_defaults.sh"
 
-# Upload all the wheels and conda packages.
+# Upload all nightly packages.
+# This has two use cases:
+# In the regular nightlies use-case this script is passed all the succesfull
+# logs
+#   ./cron/upload.sh conda_2.7_cpu.log manywheel_2.7mu_cu80.log ...
+# and only those corresponding packages are uploaded.
+# Otherwise, if given no parameters this will upload all packages.
 
-# If given package types, then only upload those package types
-if [[ "$#" -eq 0 ]]; then
-    upload_wheels=1
-    upload_condas=1
-else
-    upload_wheels=0
-    upload_condas=0
-    while [[ $# -gt 0 ]]; do
-        if [[ "$1" == *wheel* ]]; then
-            upload_wheels=1
-        fi
-        if [[ "$1" == *conda* ]]; then
-            upload_condas=1
-        fi
-        shift
-    done
-fi
+upload_it () {
+    pkg_type="$1"
+    cuda_ver="$2"
+    pkg="$3"
 
+    if [[ "$pkg_type" == 'conda' ]]; then
+        echo "Uploading $pkg_dir/$pkg to anaconda"
+        anaconda upload "$pkg_dir/$pkg" -u pytorch --label main
+    elif [[ "$pkg_type" == 'libtorch' ]]; then
+        s3_dir="s3://pytorch/libtorch/${PIP_UPLOAD_FOLDER}${cuda_ver}/"
+        echo "Uploading $pkg_dir/$pkg to $s3_dir"
+        aws s3 cp "$pkg_dir/$pkg" "$s3_dir" --acl public-read
+    else
+        uploaded_a_wheel=1
+        s3_dir="s3://pytorch/whl/${PIP_UPLOAD_FOLDER}${cuda_ver}/"
+        echo "Uploading $pkg_dir/$pkg to $s3_dir"
+        aws s3 cp "$pkg_dir/$pkg" "$s3_dir" --acl public-read
+    fi
+}
 
+# Set-up tools we need to upload
+##############################################################################
 # Source the credentials if given
 if [[ -x "$PYTORCH_CREDENTIALS_FILE" ]]; then
     source "$PYTORCH_CREDENTIALS_FILE"
@@ -61,20 +70,69 @@ if [[ "$ret" -ne 0 || "$ret1" -ne 0 || "$ret2" -ne 0 ]]; then
     yes | anaconda login --username "$PYTORCH_ANACONDA_USERNAME" --password "$PYTORCH_ANACONDA_PASSWORD"
 fi
 
+# Loop through all packages to upload
+##############################################################################
+# If given package types, then only upload those package types
+packages_to_upload=()
+if [[ "$#" -eq 0 ]]; then
+    # If not given any specific packages to upload, then upload everything that
+    # we can find
+    # Packages are organized by type and CUDA/cpu version so we have to loop
+    # over these to find all the packages
+    _ALL_PKG_TYPES=("manywheel" "wheel" "conda" "libtorch")
+    _ALL_CUDA_VERSIONS=("cpu" "cu80" "cu90" "cu92")
+    for pkg_type in "${_ALL_PKG_TYPES[@]}"; do
+        for cuda_ver in "${_ALL_CUDA_VERSIONS[@]}"; do
+            pkg_dir="$(nightlies_package_folder $pkg_type $cuda_ver)"
+            if [[ ! -d "$pkg_dir" ]]; then
+                continue
+            fi
+            all_pkgs=($(ls $pkg_dir))
+            for pkg in "${all_pkgs[@]}"; do
+                upload_it "$pkg_type" "$cuda_ver" "$pkg"
+            done
+        done
+    done
+else
+    # Else we're given a bunch of log names, turn these into exact packages
+    # Remove all '.log's from the names
+    all_configs=()
+    while [[ $# -gt 0 ]]; do
+        IFS=, confs=($(basename $1 .log | tr '_' ','))
+        pkg_type="${confs[0]}"
+        py_ver="${confs[1]}"
+        cuda_ver="${confs[2]}"
+        pkg_dir="$(nightlies_package_folder $pkg_type $cuda_ver)"
 
-# Upload wheels
-if [[ "$upload_wheels" == 1 ]]; then
-    if [[ "$(uname)" == 'Darwin' ]]; then
-        "${NIGHTLIES_BUILDER_ROOT}/wheel/upload.sh"
-    else
-        PACKAGE_ROOT_DIR="$today" "${NIGHTLIES_BUILDER_ROOT}/manywheel/upload.sh"
-    fi
+        # Map e.g. 2.7mu -> cp27mu
+        if [[ "${#py_ver}" -gt 3 ]]; then
+            if [[ "$py_ver" == '2.7mu' ]]; then
+                py_ver="cp27mu"
+            else
+                py_ver="cp${py_ver:0:1}${py_ver:2:1}m"
+            fi
+        fi
 
-    # Update wheel htmls
-    "${NIGHTLIES_BUILDER_ROOT}/update_s3_htmls.sh"
+        # Find the exact package to upload
+        # We need to match - or _ after the python version to avoid matching
+        # cp27mu when we're trying to mach cp27m. TODO this will only work when
+        # the name in both conda and wheel packages does follow the python
+        # version with a - or _
+        set +e
+        unset pkg
+        pkg="$(ls $pkg_dir | grep $py_ver[-_])"
+        set -e
+        if [[ -n "$pkg" ]]; then
+            # Add pkg_type/cuda_ver/package_name to the list
+            upload_it "$pkg_type" "$cuda_ver" "$pkg"
+        else
+            echo "!!WARNING!! Could not find the package for $1. I looked for"
+            echo "!!WARNING!! python version $py_ver in $pkg_dir but couldn't"
+            echo "!!WARNING!! find anything"
+        fi
+        shift
+    done
 fi
 
-# Upload conda packages
-if [[ "$upload_condas" == 1 ]]; then
-    "${NIGHTLIES_BUILDER_ROOT}/conda/upload.sh"
-fi
+# Update wheel htmls
+"${NIGHTLIES_BUILDER_ROOT}/update_s3_htmls.sh"