Test Bazel remote cache issue

tf_sig_build_dockerfiles/Dockerfile

@@ -7,18 +7,23 @@ COPY setup.packages.sh setup.packages.sh
 COPY builder.packages.txt builder.packages.txt
 RUN /setup.packages.sh /builder.packages.txt
 
-# Install devtoolset-7 in /dt7 with gclibc 2.12 and libstdc++ 4.4, for building
+# Install devtoolset-7 in /dt7 with glibc 2.12 and libstdc++ 4.4, for building
 # manylinux2010-compatible packages. Scripts expect to be in the root directory.
 COPY builder.devtoolset/fixlinks.sh /fixlinks.sh
 COPY builder.devtoolset/rpm-patch.sh /rpm-patch.sh
 COPY builder.devtoolset/build_devtoolset.sh /build_devtoolset.sh
 RUN /build_devtoolset.sh devtoolset-7 /dt7 
 
+# Install devtoolset-8 in /dt8 with glibc 2.17 and libstdc++ 4.8, for building
+# manylinux2014-compatible packages.
+RUN /build_devtoolset.sh devtoolset-8 /dt8 
+
 ################################################################################
 FROM nvidia/cuda:11.2.2-base-ubuntu20.04 as devel
 ################################################################################
 
 COPY --from=builder /dt7 /dt7
+COPY --from=builder /dt8 /dt8
 
 # Install required development packages but delete unneeded CUDA bloat
 # CUDA must be cleaned up in the same command to prevent Docker layer bloating

tf_sig_build_dockerfiles/builder.devtoolset/build_devtoolset.sh

@@ -15,7 +15,7 @@
 # ==============================================================================
 #
 # Builds a devtoolset cross-compiler targeting manylinux 2010 (glibc 2.12 /
-# libstdc++ 4.4).
+# libstdc++ 4.4) or manylinux2014 (glibc 2.17 / libstdc++ 4.8).
 
 VERSION="$1"
 TARGET="$2"
@@ -29,20 +29,44 @@ devtoolset-8)
   ;;
 *)
   echo "Usage: $0 {devtoolset-7|devtoolset-8} <target-directory>"
+  echo "Use 'devtoolset-7' to build a manylinux2010 compatible toolchain or 'devtoolset-8' to build a manylinux2014 compatible toolchain"
   exit 1
   ;;
 esac
 
 mkdir -p "${TARGET}"
-# Download binary glibc 2.12 release.
-wget "http://old-releases.ubuntu.com/ubuntu/pool/main/e/eglibc/libc6_2.12.1-0ubuntu6_amd64.deb" && \
-    unar "libc6_2.12.1-0ubuntu6_amd64.deb" && \
-    tar -C "${TARGET}" -xvzf "libc6_2.12.1-0ubuntu6_amd64/data.tar.gz" && \
-    rm -rf "libc6_2.12.1-0ubuntu6_amd64.deb" "libc6_2.12.1-0ubuntu6_amd64"
-wget "http://old-releases.ubuntu.com/ubuntu/pool/main/e/eglibc/libc6-dev_2.12.1-0ubuntu6_amd64.deb" && \
-    unar "libc6-dev_2.12.1-0ubuntu6_amd64.deb" && \
-    tar -C "${TARGET}" -xvzf "libc6-dev_2.12.1-0ubuntu6_amd64/data.tar.gz" && \
-    rm -rf "libc6-dev_2.12.1-0ubuntu6_amd64.deb" "libc6-dev_2.12.1-0ubuntu6_amd64"
+
+# Download glibc's shared and development libraries based on the value of the
+# `VERSION` parameter.
+# Note: 'Templatizing' this and the other conditional branches would require
+# defining several variables (version, os, path) making it difficult to maintain
+# and extend for future modifications.
+case "${VERSION}" in
+devtoolset-7)
+  # Download binary glibc 2.12 shared library release.
+  wget "http://old-releases.ubuntu.com/ubuntu/pool/main/e/eglibc/libc6_2.12.1-0ubuntu6_amd64.deb" && \
+      unar "libc6_2.12.1-0ubuntu6_amd64.deb" && \
+      tar -C "${TARGET}" -xvzf "libc6_2.12.1-0ubuntu6_amd64/data.tar.gz" && \
+      rm -rf "libc6_2.12.1-0ubuntu6_amd64.deb" "libc6_2.12.1-0ubuntu6_amd64"
+  # Download binary glibc 2.12 development library release.
+  wget "http://old-releases.ubuntu.com/ubuntu/pool/main/e/eglibc/libc6-dev_2.12.1-0ubuntu6_amd64.deb" && \
+      unar "libc6-dev_2.12.1-0ubuntu6_amd64.deb" && \
+      tar -C "${TARGET}" -xvzf "libc6-dev_2.12.1-0ubuntu6_amd64/data.tar.gz" && \
+      rm -rf "libc6-dev_2.12.1-0ubuntu6_amd64.deb" "libc6-dev_2.12.1-0ubuntu6_amd64"
+  ;;
+devtoolset-8)
+  # Download binary glibc 2.17 shared library release.
+  wget "http://old-releases.ubuntu.com/ubuntu/pool/main/e/eglibc/libc6_2.17-0ubuntu5.1_amd64.deb" && \
+      unar "libc6_2.17-0ubuntu5.1_amd64.deb" && \
+      tar -C "${TARGET}" -xvzf "libc6_2.17-0ubuntu5.1_amd64/data.tar.gz" && \
+      rm -rf "libc6_2.17-0ubuntu5.1_amd64.deb" "libc6_2.17-0ubuntu5.1_amd64"
+  # Download binary glibc 2.17 development library release.
+  wget "http://old-releases.ubuntu.com/ubuntu/pool/main/e/eglibc/libc6-dev_2.17-0ubuntu5.1_amd64.deb" && \
+      unar "libc6-dev_2.17-0ubuntu5.1_amd64.deb" && \
+      tar -C "${TARGET}" -xvzf "libc6-dev_2.17-0ubuntu5.1_amd64/data.tar.gz" && \
+      rm -rf "libc6-dev_2.17-0ubuntu5.1_amd64.deb" "libc6-dev_2.17-0ubuntu5.1_amd64"
+  ;;
+esac
 
 # Put the current kernel headers from ubuntu in place.
 ln -s "/usr/include/linux" "/${TARGET}/usr/include/linux"
@@ -56,29 +80,41 @@ ln -s "/usr/include/x86_64-linux-gnu/asm" "/${TARGET}/usr/include/asm"
 # Patch to allow non-glibc 2.12 compatible builds to work.
 sed -i '54i#define TCP_USER_TIMEOUT 18' "/${TARGET}/usr/include/netinet/tcp.h"
 
-# Download binary libstdc++ 4.4 release we are going to link against.
-# We only need the shared library, as we're going to develop against the
-# libstdc++ provided by devtoolset.
-wget "http://old-releases.ubuntu.com/ubuntu/pool/main/g/gcc-4.4/libstdc++6_4.4.3-4ubuntu5_amd64.deb" && \
-    unar "libstdc++6_4.4.3-4ubuntu5_amd64.deb" && \
-    tar -C "/${TARGET}" -xvzf "libstdc++6_4.4.3-4ubuntu5_amd64/data.tar.gz" "./usr/lib/libstdc++.so.6.0.13" && \
-    rm -rf "libstdc++6_4.4.3-4ubuntu5_amd64.deb" "libstdc++6_4.4.3-4ubuntu5_amd64"
+# Download specific version of libstdc++ shared library based on the value of
+# the `VERSION` parameter
+case "${VERSION}" in
+devtoolset-7)
+  # Download binary libstdc++ 4.4 release we are going to link against.
+  # We only need the shared library, as we're going to develop against the
+  # libstdc++ provided by devtoolset.
+  wget "http://old-releases.ubuntu.com/ubuntu/pool/main/g/gcc-4.4/libstdc++6_4.4.3-4ubuntu5_amd64.deb" && \
+      unar "libstdc++6_4.4.3-4ubuntu5_amd64.deb" && \
+      tar -C "/${TARGET}" -xvzf "libstdc++6_4.4.3-4ubuntu5_amd64/data.tar.gz" "./usr/lib/libstdc++.so.6.0.13" && \
+      rm -rf "libstdc++6_4.4.3-4ubuntu5_amd64.deb" "libstdc++6_4.4.3-4ubuntu5_amd64"
+  ;;
+devtoolset-8)
+  # Download binary libstdc++ 4.8 shared library release
+  wget "http://old-releases.ubuntu.com/ubuntu/pool/main/g/gcc-4.8/libstdc++6_4.8.1-10ubuntu8_amd64.deb" && \
+      unar "libstdc++6_4.8.1-10ubuntu8_amd64.deb" && \
+      tar -C "/${TARGET}" -xvzf "libstdc++6_4.8.1-10ubuntu8_amd64/data.tar.gz" "./usr/lib/libstdc++.so.6.0.18" && \
+      rm -rf "libstdc++6_4.8.1-10ubuntu8_amd64.deb" "libstdc++6_4.8.1-10ubuntu8_amd64"
+  ;;
+esac
 
 mkdir -p "${TARGET}-src"
 cd "${TARGET}-src"
 
-# Build a devtoolset cross-compiler based on our glibc 2.12 sysroot setup.
-
+# Build a devtoolset cross-compiler based on our glibc 2.12/glibc 2.17 sysroot setup.
 case "${VERSION}" in
 devtoolset-7)
   wget "http://vault.centos.org/centos/6/sclo/Source/rh/devtoolset-7/devtoolset-7-gcc-7.3.1-5.15.el6.src.rpm"
   rpm2cpio "devtoolset-7-gcc-7.3.1-5.15.el6.src.rpm" |cpio -idmv
   tar -xvjf "gcc-7.3.1-20180303.tar.bz2" --strip 1
   ;;
 devtoolset-8)
-  wget "http://vault.centos.org/centos/6/sclo/Source/rh/devtoolset-8/devtoolset-8-gcc-8.2.1-3.el6.src.rpm"
-  rpm2cpio "devtoolset-8-gcc-8.2.1-3.el6.src.rpm" |cpio -idmv
-  tar -xvf "gcc-8.2.1-20180905.tar.xz" --strip 1
+  wget "https://vault.centos.org/centos/7/sclo/Source/rh/devtoolset-8-gcc-8.3.1-3.el7.src.rpm"
+  rpm2cpio "devtoolset-8-gcc-8.3.1-3.el7.src.rpm" |cpio -idmv
+  tar -xvf "gcc-8.3.1-20190311.tar.xz" --strip 1
   ;;
 esac
 
@@ -117,14 +153,30 @@ cd "${TARGET}-build"
     make -j 42 && \
     make install
 
+
 # Create the devtoolset libstdc++ linkerscript that links dynamically against
 # the system libstdc++ 4.4 and provides all other symbols statically.
+case "${VERSION}" in
+devtoolset-7)
 mv "/${TARGET}/usr/lib/libstdc++.so.${LIBSTDCXX_VERSION}" \
    "/${TARGET}/usr/lib/libstdc++.so.${LIBSTDCXX_VERSION}.backup"
 echo -e "OUTPUT_FORMAT(elf64-x86-64)\nINPUT ( libstdc++.so.6.0.13 -lstdc++_nonshared44 )" \
    > "/${TARGET}/usr/lib/libstdc++.so.${LIBSTDCXX_VERSION}"
 cp "./x86_64-pc-linux-gnu/libstdc++-v3/src/.libs/libstdc++_nonshared44.a" \
    "/${TARGET}/usr/lib"
+  ;;
+devtoolset-8)
+# Note that the installation path for libstdc++ here is /${TARGET}/usr/lib64/
+mv "/${TARGET}/usr/lib64/libstdc++.so.${LIBSTDCXX_VERSION}" \
+   "/${TARGET}/usr/lib64/libstdc++.so.${LIBSTDCXX_VERSION}.backup"
+echo -e "OUTPUT_FORMAT(elf64-x86-64)\nINPUT ( libstdc++.so.6.0.18 -lstdc++_nonshared48 )" \
+   > "/${TARGET}/usr/lib/libstdc++.so.${LIBSTDCXX_VERSION}"
+cp "./x86_64-pc-linux-gnu/libstdc++-v3/src/.libs/libstdc++_nonshared48.a" \
+   "/${TARGET}/usr/lib"
+cp "./x86_64-pc-linux-gnu/libstdc++-v3/src/.libs/libstdc++_nonshared48.a" \
+   "/${TARGET}/usr/lib64"
+  ;;
+esac
 
 # Link in architecture specific includes from the system; note that we cannot
 # link in the whole x86_64-linux-gnu folder, as otherwise we're overlaying
@@ -136,4 +188,3 @@ PYTHON_VERSIONS=("python3.7m" "python3.8" "python3.9" "python3.10")
 for v in "${PYTHON_VERSIONS[@]}"; do
   ln -s "/usr/local/include/${v}" "/${TARGET}/usr/include/x86_64-linux-gnu/${v}"
 done
-

tf_sig_build_dockerfiles/devel.packages.txt

@@ -30,6 +30,7 @@ clang-format-12
 colordiff
 curl
 ffmpeg
+gdb
 git
 jq
 less

tf_sig_build_dockerfiles/devel.usertools/cpu.bazelrc

@@ -32,9 +32,14 @@ build --copt=-mavx --host_copt=-mavx
 # See https://docs.bazel.build/versions/main/skylark/performance.html#performance-profiling
 build --profile=/tf/pkg/profile.json
 
-# Use the NVCC toolchain to compile for manylinux2010
-build [email protected]gcc7_manylinux2010-cuda11.2-cudnn8.1-tensorrt7.2_config_cuda//crosstool:toolchain
+# Use the NVCC toolchain to compile for manylinux2014
+build [email protected]gcc8_manylinux2014-cuda11.2-cudnn8.1-tensorrt7.2_config_cuda//crosstool:toolchain
 
+# set cachebust to manylinux2014
+# build --action_env=CACHEBUST=manylinux2014
+
+# add remote cache header
+build --remote_cache_header=is_manylinux2014=true
 # Test-related settings below this point.
 test --build_tests_only --keep_going --test_output=errors --verbose_failures=true
 test --local_test_jobs=HOST_CPUS
@@ -48,8 +53,10 @@ test --test_summary=terse
 # bazel test invocation as normal.
 test:nonpip --test_tag_filters=-no_oss,-oss_serial,-gpu,-tpu,-benchmark-test,-v1only
 test:nonpip --build_tag_filters=-no_oss,-oss_serial,-gpu,-tpu,-benchmark-test,-v1only
-test:nonpip --test_lang_filters=py --flaky_test_attempts=3 --test_size_filters=small,medium
-test:nonpip -- //tensorflow/... -//tensorflow/python/integration_testing/... -//tensorflow/compiler/tf2tensorrt/... -//tensorflow/compiler/xrt/... -//tensorflow/core/tpu/... -//tensorflow/lite/...
+# test:nonpip -- //tensorflow/... -//tensorflow/python/integration_testing/... -//tensorflow/compiler/tf2tensorrt/... -//tensorflow/compiler/xrt/... -//tensorflow/core/tpu/... -//tensorflow/lite/...
+
+# Test a specific test that should be failing
+test:nonpip -- //tensorflow/python:histogram_ops_test_cpu
 
 # "pip tests" run a similar suite of tests the "nonpip" tests, but do something
 # odd to attempt to validate the quality of the pip package. The wheel is
@@ -72,9 +79,7 @@ test:pip --config=pip_venv
 # Yes, we don't exclude the gpu tests on pip for some reason.
 test:pip --test_tag_filters=-nopip,-no_pip,-no_oss,-oss_serial,-v1only
 test:pip --build_tag_filters=-nopip,-no_pip,-no_oss,-oss_serial,-v1only
-test:pip --test_lang_filters=py --flaky_test_attempts=3 --test_size_filters=small,medium
-test:pip -- //bazel_pip/tensorflow/... -//bazel_pip/tensorflow/python/integration_testing/... -//bazel_pip/tensorflow/compiler/tf2tensorrt/... -//bazel_pip/tensorflow/compiler/xrt/... -//bazel_pip/tensorflow/core/tpu/... -//bazel_pip/tensorflow/lite/...
-
-# For building libtensorflow archives
-test:libtensorflow_test -- //tensorflow/tools/lib_package:libtensorflow_test //tensorflow/tools/lib_package:libtensorflow_java_test
-build:libtensorflow_build -- //tensorflow/tools/lib_package:libtensorflow.tar.gz //tensorflow/tools/lib_package:libtensorflow_jni.tar.gz //tensorflow/java:libtensorflow.jar //tensorflow/java:libtensorflow-src.jar //tensorflow/tools/lib_package:libtensorflow_proto.zip
+test:pip --define=no_tensorflow_py_deps=true
+# Test a specific test that should be failing
+test:pip -- //bazel_pip/tensorflow/python:histogram_ops_test_cpu
+# test:pip -- //bazel_pip/tensorflow/... -//bazel_pip/tensorflow/python/integration_testing/... -//bazel_pip/tensorflow/compiler/tf2tensorrt/... -//bazel_pip/tensorflow/compiler/xrt/... -//bazel_pip/tensorflow/core/tpu/... -//bazel_pip/tensorflow/lite/...

tf_sig_build_dockerfiles/devel.usertools/gpu.bazelrc

@@ -38,9 +38,9 @@ build --repo_env TF_NEED_CUDA=1
 build --action_env=TF_CUDA_VERSION="11"
 build --action_env=TF_CUDNN_VERSION="8"
 build --action_env=CUDA_TOOLKIT_PATH="/usr/local/cuda-11.2"
-build --action_env=GCC_HOST_COMPILER_PATH="/dt7/usr/bin/gcc"
+build --action_env=GCC_HOST_COMPILER_PATH="/dt8/usr/bin/gcc"
 build --action_env=LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/tensorrt/lib"
-build [email protected]gcc7_manylinux2010-cuda11.2-cudnn8.1-tensorrt7.2_config_cuda//crosstool:toolchain
+build [email protected]gcc8_manylinux2014-cuda11.2-cudnn8.1-tensorrt7.2_config_cuda//crosstool:toolchain
 
 # CUDA: Enable TensorRT optimizations
 # https://developer.nvidia.com/tensorrt

tf_sig_build_dockerfiles/devel.usertools/rename_and_verify_wheels.sh

@@ -6,7 +6,7 @@ set -euxo pipefail
 
 for wheel in /tf/pkg/*.whl; do
   echo "Checking and renaming $wheel..."
-  time python3 -m auditwheel repair --plat manylinux2010_x86_64 "$wheel" --wheel-dir /tf/pkg 2>&1 | tee check.txt
+  time python3 -m auditwheel repair --plat manylinux2014_x86_64 "$wheel" --wheel-dir /tf/pkg 2>&1 | tee check.txt
 
   # We don't need the original wheel if it was renamed
   new_wheel=$(grep --extended-regexp --only-matching '/tf/pkg/\S+.whl' check.txt)

tf_sig_build_dockerfiles/devel.usertools/wheel_verification.bats

@@ -12,9 +12,9 @@ teardown_file() {
     rm -rf /tf/venv
 }
 
-@test "Wheel is manylinux2010 (manylinux_2_12) compliant" {
+@test "Wheel is manylinux2014 (manylinux_2_17) compliant" {
     python3 -m auditwheel show "$TF_WHEEL" > audit.txt
-    grep --quiet 'This constrains the platform tag to "manylinux_2_12_x86_64"' audit.txt
+    grep --quiet 'This constrains the platform tag to "manylinux_2_17_x86_64"' audit.txt
 }
 
 @test "Wheel conforms to upstream size limitations" {

tf_sig_build_dockerfiles/setup.python.sh

@@ -21,8 +21,12 @@ EOF
 # for any Python version present
 pushd /usr/include/x86_64-linux-gnu
 for f in $(ls | grep python); do
+  # set up symlink for devtoolset-7
   rm -f /dt7/usr/include/x86_64-linux-gnu/$f
   ln -s /usr/include/x86_64-linux-gnu/$f /dt7/usr/include/x86_64-linux-gnu/$f
+  # set up symlink for devtoolset-8
+  rm -f /dt8/usr/include/x86_64-linux-gnu/$f
+  ln -s /usr/include/x86_64-linux-gnu/$f /dt8/usr/include/x86_64-linux-gnu/$f
 done
 popd