Install static curl

ejguan · ejguan · commit 21abff473533 · 2022-05-24T19:10:24.000Z
diff --git a/.github/workflows/_build_test_upload.yml b/.github/workflows/_build_test_upload.yml
@@ -87,11 +87,16 @@ jobs:
           arch: x64
       - name: Install Build Dependency
         shell: bash
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
         run: |
           if ${{ startsWith( matrix.os, 'ubuntu' ) }}; then
-            yum -y install ninja-build
-            # yum -y install openssl-devel openssl-static curl-devel zlib-static
-            yum -y install curl-devel zlib-static
+            source packaging/manylinux/python_helper.sh
+            yum -y install ninja-build zlib-static
+            # Docker path is /__w by default
+            export WORKSPACE="/__w"
+            # Install static OpenSSL/libcrypto library
+            ./packaging/manylinux/install_openssl_curl.sh
           else
             pip install cmake ninja
             echo "/home/runner/.local/bin" >> $GITHUB_PATH
@@ -105,16 +110,13 @@ jobs:
         run: |
           if ${{ startsWith( matrix.os, 'ubuntu' ) }}; then
             source packaging/manylinux/python_helper.sh
-            # Docker path is /__w by default
-            export WORKSPACE="/__w"
             # See: https://github.com/actions/checkout/issues/760
             git config --global --add safe.directory "$WORKSPACE/data/data"
-            # Install static OpenSSL/libcrypto library
-            ./packaging/manylinux/install_openssl.sh
             # AWSSDK uses $CMAKE_PREFIX_PATH to find openssl
-            export CMAKE_PREFIX_PATH="$WORKSPACE/ssl/:$CMAKE_PREFIX_PATH"
-            export OPENSSL_ROOT_DIR="$WORKSPACE/ssl"
-            export STATIC_OPENSSL=TRUE
+            export OPENSSL_ROOT_DIR="/__w/ssl"
+            export CURL_ROOT_DIR="/__w/curl"
+            export CMAKE_PREFIX_PATH="$OPENSSL_ROOT_DIR:$CURL_ROOT_DIR:$CMAKE_PREFIX_PATH"
+            export STATIC_DEPS=TRUE
           fi
           packaging/build_wheel.sh
       - name: Validate TorchData Wheel
@@ -211,7 +213,7 @@ jobs:
   #           --username __token__ \
   #           --password "$PYPI_TOKEN" \
   #           dist/torchdata*.whl
-  # 
+  #
   # conda_build_test:
   #   needs: get_release_type
   #   runs-on: ${{ matrix.os }}
@@ -269,7 +271,7 @@ jobs:
   #       with:
   #         name: torchdata-artifact
   #         path: conda-bld/*/torchdata-*.tar.bz2
-  # 
+  #
   # conda_upload:
   #   if: always() && inputs.branch != ''
   #   needs: [get_release_type, conda_build_test]
@@ -304,13 +306,13 @@ jobs:
   #       run: |
   #         conda install -yq anaconda-client
   #         conda install -c conda-forge -yq jq
-  #     
+  #
   #         if [[ ${{ needs.get_release_type.outputs.type }} == 'official' ]]; then
   #           CONDA_CHANNEL=pytorch
   #         else
   #           CONDA_CHANNEL=pytorch-${{ needs.get_release_type.outputs.type }}
   #         fi
-  #     
+  #
   #         if [[ ${{ needs.get_release_type.outputs.type }} == 'nightly' ]]; then
   #           # Loop over all platforms [win-64, osx-64, linux-64]
   #           for subdir in $(find . -type f -name '*torchdata*.tar.bz2' | sed -r 's|/[^/]+$||' | uniq | cut -f2 -d/) ; do
@@ -339,7 +341,7 @@ jobs:
   #         else
   #           anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ./*/torchdata-*.tar.bz2 -u "$CONDA_CHANNEL" --label main --no-progress --force
   #         fi
-  # 
+  #
   # build_docs:
   #   if: |
   #     always() && inputs.branch != '' &&
@@ -373,14 +375,14 @@ jobs:
   #         else
   #           PYTORCH_VERSION=torch==${{ inputs.pytorch_version }}
   #         fi
-  # 
+  #
   #         PIP_CHANNEL=${{ needs.get_release_type.outputs.type }}
   #         if [[ $PIP_CHANNEL == 'official' ]]; then
   #           pip3 install "$PYTORCH_VERSION" -f https://download.pytorch.org/whl/torch_stable.html
   #         else
   #           pip3 install --pre "$PYTORCH_VERSION" -f "https://download.pytorch.org/whl/$PIP_CHANNEL/torch_$PIP_CHANNEL.html"
   #         fi
-  # 
+  #
   #         pip3 install -r requirements.txt
   #         python3 setup.py install
   #     - name: Check env
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -4,13 +4,13 @@ on:
     branches:
       - main
       - release/*
-  # pull_request:
-  #   types: [opened, synchronize, reopened, labeled]
-  #   branches:
-  #     - main
-  #     # For PR created by ghstack
-  #     - gh/*/*/base
-  #     - release/*
+  pull_request:
+    types: [opened, synchronize, reopened, labeled]
+    branches:
+      - main
+      # For PR created by ghstack
+      - gh/*/*/base
+      - release/*
 
 jobs:
   test:
@@ -29,9 +29,6 @@ jobs:
           - 3.7
           - 3.8
           - 3.9
-        with-s3:
-          - 1
-          - 0
     steps:
       - name: Setup additional system libraries
         if: startsWith( matrix.os, 'ubuntu' )
@@ -44,10 +41,10 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - name: Setup msbuild on Windows
-        if: matrix.with-s3 == 1 && matrix.os == 'windows-latest'
+        if: matrix.os == 'windows-latest'
         uses: microsoft/setup-msbuild@v1.1
       - name: Set up Visual Studio shell
-        if: matrix.with-s3 == 1 && matrix.os == 'windows-latest'
+        if: matrix.os == 'windows-latest'
         uses: egor-tensin/vs-shell@v2
         with:
           arch: x64
@@ -65,7 +62,7 @@ jobs:
         run: |
           python setup.py install
         env:
-          BUILD_S3: ${{ matrix.with-s3 }}
+          BUILD_S3: 1
       - name: Install test requirements
         run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile protobuf
       - name: Run DataPipes tests with pytest
diff --git a/.github/workflows/domain_ci.yml b/.github/workflows/domain_ci.yml
@@ -4,12 +4,12 @@ on:
     branches:
       - main
       - release/*
-  # pull_request:
-  #   branches:
-  #     - main
-  #     # For PR created by ghstack
-  #     - gh/*/*/base
-  #     - release/*
+  pull_request:
+    branches:
+      - main
+      # For PR created by ghstack
+      - gh/*/*/base
+      - release/*
 
 jobs:
   torchvision:
diff --git a/packaging/manylinux/install_openssl_curl.sh b/packaging/manylinux/install_openssl_curl.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+OPENSSL_URL="https://www.openssl.org/source/"
+OPENSSL_NAME="openssl-1.1.1o"
+OPENSSL_SHA256="9384a2b0570dd80358841464677115df785edb941c71211f75076d72fe6b438f"
+OPENSSL_BUILD_FLAGS="no-ssl2 no-zlib no-shared no-comp no-dynamic-engine enable-ec_nistp_64_gcc_128"
+
+CURL_URL="https://github.com/curl/curl/releases/download"
+CURL_NAME="curl-7.83.1"
+CURL_BUILD_FLAGS="--disable-shared"
+
+function check_sha256sum {
+    local fname=$1
+    local sha256=$2
+    echo "${sha256}  ${fname}" > ${fname}.sha256
+    sha256sum -c ${fname}.sha256
+    rm ${fname}.sha256
+}
+
+yum erase -y openssl-devel curl-devel
+
+pushd ${WORKSPACE}
+
+# OpenSSL
+curl -fsSL -o ${OPENSSL_NAME}.tar.gz ${OPENSSL_URL}/${OPENSSL_NAME}.tar.gz
+check_sha256sum ${OPENSSL_NAME}.tar.gz ${OPENSSL_SHA256}
+tar zxf ${OPENSSL_NAME}.tar.gz
+
+pushd ${OPENSSL_NAME}
+
+./config $OPENSSL_BUILD_FLAGS --prefix=${WORKSPACE}/ssl --openssldir=${WORKSPACE}/ssl
+make -j4 > /dev/null
+# avoid installing the docs
+# https://github.com/openssl/openssl/issues/6685#issuecomment-403838728
+make install_sw > /dev/null
+
+popd
+rm -rf ${OPENSSL_NAME} ${OPENSSL_NAME}.tar.gz
+
+# cURL
+curl -fsSL -o ${CURL_NAME}.tar.gz ${CURL_URL}/${CURL_NAME//./_}/${CURL_NAME}.tar.gz
+tar zxf ${CURL_NAME}.tar.gz
+
+pushd ${CURL_NAME}
+
+./configure ${CURL_BUILD_FLAGS} --with-openssl=${WORKSPACE}/ssl --prefix=${WORKSPACE}/curl 
+make -j4 > /dev/null
+make install > /dev/null
+
+popd
+rm -rf ${CURL_NAME} ${CURL_NAME}.tar.gz
+
+popd
diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt
@@ -24,14 +24,16 @@ if(BUILD_S3)
   elseif(APPLE)
     set(AWSSDK_DEP_LIBRARIES pthread curl)
   elseif(UNIX)
-    if(STATIC_OPENSSL)
+    set(AWSSDK_DEP_LIBRARIES pthread)
+    if(STATIC_DEPS)
       set(OPENSSL_USE_STATIC_LIBS TRUE)
     endif()
+    include(FindCURL)
+    list(APPEND AWSSDK_DEP_LIBRARIES ${CURL_LIBRARIES})
     include(FindOpenSSL)
-    set(AWSSDK_DEP_INCLUDE_DIRS ${AWSSDK_INCLUDE_DIRS})
-    set(AWSSDK_DEP_LIBRARIES ${OPENSSL_CRYPTO_LIBRARIES} ${OPENSSL_SSL_LIBRARIES} curl)
+    list(APPEND AWSSDK_DEP_LIBRARIES ${OPENSSL_SSL_LIBRARIES} ${OPENSSL_CRYPTO_LIBRARIES})
   endif()
-  message(STATUS "AWSSDK DEPENDENCIES AWSSDK_DEP_INCLUDE_DIRS: ${AWSSDK_DEP_INCLUDE_DIRS}, AWSSDK_DEP_LIBRARIES: ${AWSSDK_DEP_LIBRARIES}")
+  message(STATUS "AWSSDK DEPENDENCIES AWSSDK_DEP_LIBRARIES: ${AWSSDK_DEP_LIBRARIES}")
 
   set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
@@ -54,7 +56,6 @@ if(BUILD_S3)
     ${Python_INCLUDE_DIR}
     ${pybind11_INCLUDE_DIRS}
     ${AWSSDK_INCLUDE_DIRS}
-    ${AWSSDK_DEP_INCLUDE_DIRS}
     )
 
   target_link_libraries(
@@ -66,13 +67,6 @@ if(BUILD_S3)
     ${AWSSDK_DEP_LIBRARIES}
     )
 
-  if(UNIX)
-    target_link_libraries(
-      _torchdata
-      PRIVATE
-    )
-  endif()
-
   set_target_properties(_torchdata PROPERTIES PREFIX "")
   if (MSVC)
     set_target_properties(_torchdata PROPERTIES SUFFIX ".pyd")
