diff --git a/tools/build_pytorch_libs.sh b/tools/build_pytorch_libs.sh
index 4a0dbd04c905f1..8f79c2830e96c0 100755
--- a/tools/build_pytorch_libs.sh
+++ b/tools/build_pytorch_libs.sh
@@ -97,7 +97,11 @@ if [[ $(uname) == 'Darwin' ]]; then
     LDFLAGS="$LDFLAGS -Wl,-rpath,@loader_path"
     LD_POSTFIX=".dylib"
 else
-    LDFLAGS="$LDFLAGS -Wl,-rpath,\$ORIGIN"
+    if [[ $USE_ROCM -eq 1 ]]; then
+        LDFLAGS="$LDFLAGS -Wl,-rpath,\\\\\\\$ORIGIN"
+    else
+        LDFLAGS="$LDFLAGS -Wl,-rpath,\$ORIGIN"
+    fi
 fi
 CPP_FLAGS=" -std=c++11 "
 GLOO_FLAGS=""