Merge branch 'master' into ck-631

Author: chayim

.circleci/config.yml

@@ -14,7 +14,7 @@ commands:
       - run:
           name: Avoid tests for docs
           command: |
-            if [[ $CIRCLE_BRANCH == *docs* ]]; then
+            if [[ $CIRCLE_BRANCH == *docs ]]; then
               echo "Identifies as documents PR, no testing required"
               circleci step halt
             fi
@@ -48,12 +48,12 @@ commands:
       - run:
           name: Relocate docker overlay2 dir
           command: |
-            sudo service docker stop
+            sudo systemctl stop docker
             sudo mkdir -p /var2/lib/docker
             sudo mv /var/lib/docker/overlay2 /var2/lib/docker
             sudo mkdir /var/lib/docker/overlay2
             sudo mount --bind /var2/lib/docker/overlay2 /var/lib/docker/overlay2
-            sudo service docker start
+            sudo systemctl start docker
 
   setup-automation:
     steps:
@@ -129,8 +129,8 @@ commands:
   platforms-build-steps:
     steps:
       - abort_for_docs
-      - setup-executor
       - checkout-all
+      - relocate-docker-storage
       - setup-automation
       - run:
           name: Build for platform
@@ -199,11 +199,12 @@ jobs:
           platform: debian
 
   platforms-build:
-    docker:
-      - image: cimg/base:2020.01
+    machine:
+      enabled: true
+      docker_layer_caching: true
+      resource_class: gpu.nvidia.small
+      image: ubuntu-1604-cuda-11.1:202012-01
     steps:
-      - setup-executor
-      - relocate-docker-storage
       - platforms-build-steps
 
   coverage:
@@ -294,7 +295,6 @@ jobs:
             make -C opt test VALGRIND=1 <<parameters.test_args>>
           no_output_timeout: 120m
 
-
 #  build-macos:
 #    macos:
 #      xcode: 11.3.0

.github/workflows/trigger-build-and-test-gpu.yaml

@@ -10,7 +10,7 @@ jobs:
   valgrind_general:
     if: github.event.label.name == 'ci-test'
     runs-on: ubuntu-latest
-    name: Run build and test cpu
+    name: Run build and test gpu
 
     steps:
       - uses: actions/checkout@v2

opt/clang-check-all.sh

@@ -4,8 +4,14 @@
 set -e
 
 CLANG_FMT_SRCS=$(find ../src/ \( -name '*.c' -o -name '*.cc'-o -name '*.cpp' -o -name '*.h' -o -name '*.hh' -o -name '*.hpp' \))
+CLANG_FMT_TESTS=$(find ../tests/ \( -name '*.c' -o -name '*.cc'-o -name '*.cpp' -o -name '*.h' -o -name '*.hh' -o -name '*.hpp' \))
 
 for filename in $CLANG_FMT_SRCS; do
     echo "Checking $filename"
     clang-format -style=file -Werror --dry-run $filename
 done
+
+for filename in $CLANG_FMT_TESTS; do
+    echo "Checking $filename"
+    clang-format -style=file -Werror --dry-run $filename
+done

opt/clang-format-all.sh

@@ -4,7 +4,12 @@
 set -e
 
 CLANG_FMT_SRCS=$(find ../src/ \( -name '*.c' -o -name '*.cc'-o -name '*.cpp' -o -name '*.h' -o -name '*.hh' -o -name '*.hpp' \))
+CLANG_FMT_TESTS=$(find ../tests/ \( -name '*.c' -o -name '*.cc'-o -name '*.cpp' -o -name '*.h' -o -name '*.hh' -o -name '*.hpp' \))
 
 for filename in $CLANG_FMT_SRCS; do
     clang-format --verbose -style=file -i $filename
 done
+
+for filename in $CLANG_FMT_TESTS; do
+    clang-format --verbose -style=file -i $filename
+done

tests/flow/pt_api_test.c

@@ -4,41 +4,41 @@
 #include "dlpack/dlpack.h"
 
 int main() {
-  printf("Hello from libTorch C library version %s\n", "1.0");
-  torchBasicTest();
+    printf("Hello from libTorch C library version %s\n", "1.0");
+    torchBasicTest();
 
-  void* ctx;
-  char *err = NULL;
+    void *ctx;
+    char *err = NULL;
 
-  const char script[] = "\n\
+    const char script[] = "\n\
     def foo(a):          \n\
         return a * 2     \n\
     ";
 
-  ctx = torchCompileScript(script, kDLCPU, &err);
-  if (err) {
-    printf("ERR: %s\n", err);
-    free(err);
-    err = NULL;
-    return 1;
-  }
-  printf("Compiled: %p\n", ctx);
+    ctx = torchCompileScript(script, kDLCPU, &err);
+    if (err) {
+        printf("ERR: %s\n", err);
+        free(err);
+        err = NULL;
+        return 1;
+    }
+    printf("Compiled: %p\n", ctx);
 
-  DLDataType dtype = (DLDataType){ .code = kDLFloat, .bits = 32, .lanes = 1};
-  int64_t shape[1] = {1};
-  int64_t strides[1] = {1};
-  char data[4] = "0000";
-  DLManagedTensor* input = torchNewTensor(dtype, 1, shape, strides, data);
-  DLManagedTensor* output;
-  torchRunScript(ctx, "foo", 1, &input, 1, &output, &err);
-  if (err) {
-    printf("ERR: %s\n", err);
-    free(err);
-    return 1;
-  }
+    DLDataType dtype = (DLDataType){.code = kDLFloat, .bits = 32, .lanes = 1};
+    int64_t shape[1] = {1};
+    int64_t strides[1] = {1};
+    char data[4] = "0000";
+    DLManagedTensor *input = torchNewTensor(dtype, 1, shape, strides, data);
+    DLManagedTensor *output;
+    torchRunScript(ctx, "foo", 1, &input, 1, &output, &err);
+    if (err) {
+        printf("ERR: %s\n", err);
+        free(err);
+        return 1;
+    }
 
-  torchDeallocContext(ctx);
-  printf("Deallocated\n");
+    torchDeallocContext(ctx);
+    printf("Deallocated\n");
 
-  return 0;
+    return 0;
 }

tests/flow/tests_llapi.py

@@ -101,6 +101,12 @@ def test_dag_build_and_run(env):
     ret = con.execute_command("RAI_llapi.DAGrun")
     env.assertEqual(ret, b'DAG run success')
 
+    # Run the DAG LLAPI test again with multi process test to ensure that there are no dead-locks
+    def run_dag_llapi(con):
+        con.execute_command("RAI_llapi.DAGrun")
+
+    run_test_multiproc(env, 500, run_dag_llapi)
+
 
 @with_test_module
 def test_dagrun_multidevice_resnet(env):

tests/flow/tf_api_test.c

@@ -2,6 +2,6 @@
 #include <tensorflow/c/c_api.h>
 
 int main() {
-  printf("Hello from TensorFlow C library version %s\n", TF_Version());
-  return 0;
+    printf("Hello from TensorFlow C library version %s\n", TF_Version());
+    return 0;
 }

tests/module/DAG_utils.c

@@ -3,13 +3,15 @@
 #include <pthread.h>
 #include <stdbool.h>
 
-#define LLAPIMODULE_OK 0
+#define LLAPIMODULE_OK  0
 #define LLAPIMODULE_ERR 1
 
-typedef struct RAI_RunResults {
+typedef struct RAI_RunCtx {
     RAI_Tensor **outputs;
     RAI_Error *error;
-} RAI_RunResults;
+    pthread_mutex_t lock;
+    pthread_cond_t cond;
+} RAI_RunCtx;
 
 int testLoadTensor(RedisModuleCtx *ctx);