v5.0.x accelerator/cuda: Add delayed initialization logic

.ci/README.md

@@ -1,16 +1,4 @@
-# Open MPI Continuous Integration (CI) Services
-## Mellanox Open MPI CI
-### Scope
-[Mellanox](https://www.mellanox.com/) Open MPI CI is intended to verify Open MPI with recent Mellanox SW components ([Mellanox OFED](https://www.mellanox.com/page/products_dyn?product_family=26), [UCX](https://www.mellanox.com/page/products_dyn?product_family=281&mtag=ucx) and other [HPC-X](https://www.mellanox.com/page/products_dyn?product_family=189&mtag=hpc-x) components) in the Mellanox lab environment.
+Top-level directory for CI tests.
 
-CI is managed by [Azure Pipelines](https://docs.microsoft.com/en-us/azure/devops/pipelines/?view=azure-devops) service.
-
-Mellanox Open MPI CI includes:
-* Open MPI building with internal stable engineering versions of UCX and HCOLL. The building is run in Docker-based environment.
-* Sanity functional testing.
-### How to Run CI
-Mellanox Open MPI CI is triggered upon the following events:
-* Create a pull request (PR). CI status is visible in the PR status. CI is restarted automatically upon each new commit within the PR. CI status and log files are also available on the Azure DevOps server.
-* Trigger CI with special PR comments (for example, `/azp run`). Comment triggers are available only if the comment author has write permission to the PR target repo. Detailed information about comment triggers is available in the official Azure DevOps [documentation](https://docs.microsoft.com/en-us/azure/devops/pipelines/repos/github?view=azure-devops&tabs=yaml#comment-triggers).
-### Support
-In case of any issues, questions or suggestions please contact to [Mellanox Open MPI CI support team](mailto:[email protected]).
+Feel free to make your own subdirectory (e.g., for your organization)
+and put CI tests and supporting infrastructure here.

.ci/lanl/gitlab-darwin-ci.yml

@@ -0,0 +1,301 @@
+variables:
+  SCHEDULER_PARAMETERS: "-pgeneral -t 4:00:00 -N 1 --ntasks-per-node=16"
+  GIT_STRATEGY: clone
+  NPROCS: 4
+
+stages:
+  - build
+  - test
+
+build:intel:
+  stage: build
+  tags: [darwin-slurm-shared]
+  script:
+    - module load intel/2022.0.1
+    - rm .gitmodules
+    - cp $GITSUBMODULEPATCH .gitmodules
+    - git submodule update --init --recursive
+    - ./autogen.pl
+    - ./configure CC=icx FC=ifx CXX=icpx --prefix=$PWD/install_test --with-libevent=internal
+    - make -j 8 install
+    - make check
+    - export PATH=$PWD/install_test/bin:$PATH
+    - cd examples 
+    - make
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    untracked: true
+    paths:
+      - examples
+      - install_test
+    expire_in: 1 week
+
+build:ibm:
+  stage: build
+  tags: [darwin-slurm-shared]
+  variables:
+    SCHEDULER_PARAMETERS: "-ppower9 -t 4:00:00 -N 1 --ntasks-per-node=16"
+  script:
+    - module load ibm
+    - rm .gitmodules
+    - cp $GITSUBMODULEPATCH .gitmodules
+    - git submodule update --init --recursive
+    - ./autogen.pl
+    - ./configure CC=xlc FC=xlf CXX=xlc++ --prefix=$PWD/install_test --with-libevent=internal
+    - make -j 8 install
+    - make check
+    - export PATH=$PWD/install_test/bin:$PATH
+    - cd examples
+    - make
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    untracked: true
+    paths:
+      - examples
+      - install_test
+    expire_in: 1 week
+
+build:amd:
+  stage: build
+  tags: [darwin-slurm-shared]
+  variables:
+    SCHEDULER_PARAMETERS: "-pamd-rome -t 4:00:00 -N 1 --ntasks-per-node=16"
+  script:
+    - module load aocc/3.0.0
+    - rm .gitmodules
+    - cp $GITSUBMODULEPATCH .gitmodules
+    - git submodule update --init --recursive
+    - ./autogen.pl
+    - ./configure CC=clang FC=flang CXX=clang++ --prefix=$PWD/install_test --with-libevent=internal LIBS="-lucm -lucs"
+    - make -j 8 install
+    - make check
+    - export PATH=$PWD/install_test/bin:$PATH
+    - cd examples
+    - make
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    untracked: true
+    paths:
+      - examples
+      - install_test
+    expire_in: 1 week
+
+build:gnu:
+  stage: build
+  tags: [darwin-slurm-shared]
+  script:
+    - module load gcc
+    - rm .gitmodules
+    - cp $GITSUBMODULEPATCH .gitmodules
+    - git submodule update --init --recursive
+    - ./autogen.pl
+    - ./configure --prefix=$PWD/install_test --with-libevent=internal
+    - make -j 8 install
+    - make check
+    - export PATH=$PWD/install_test/bin:$PATH
+    - cd examples 
+    - make
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    untracked: true
+    paths:
+      - examples
+      - install_test
+    expire_in: 1 week
+
+build:cce:
+  stage: build
+  tags: [darwin-slurm-shared]
+  variables:
+    SCHEDULER_PARAMETERS: "-pcrossroads-dev -t 4:00:00 -N 1 --ntasks-per-node=16"
+  script:
+    - hostname
+    - module use --append /opt/cray/pe/modulefiles
+    - module avail cce
+    - module load cce
+    - module unload libsci
+    - module unload cray-mvapich2_nogpu
+    - rm .gitmodules
+    - cp $GITSUBMODULEPATCH .gitmodules
+    - git submodule update --init --recursive
+    - ./autogen.pl
+    - ./configure CC=cc FTN=ftn CXX=CC --with-ucx=no --prefix=$PWD/install_test --with-libevent=internal
+    - make -j 8 install
+    - make check
+    - export PATH=$PWD/install_test/bin:$PATH
+    - cd examples
+    - make
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    untracked: true
+    paths:
+      - examples
+      - install_test
+    expire_in: 1 week
+
+
+test:intel:
+  stage: test
+  tags: [darwin-slurm-shared]
+  dependencies:
+    - build:intel
+  needs: ["build:intel"]
+  script:
+    - pwd
+    - ls
+    - module load intel/2022.0.1
+    - export PATH=$PWD/install_test/bin:$PATH
+    - which mpirun
+    - cd examples
+    - mpirun -np 4 hostname
+    - mpirun -np 4 ./hello_c
+    - mpirun -np 4 ./ring_c
+    - mpirun -np 4 ./hello_mpifh
+    - mpirun -np 4 ./ring_mpifh
+    - mpirun -np 4 ./hello_usempi
+    - mpirun -np 4 ./ring_usempi
+    - mpirun -np 4 ./hello_usempif08
+    - mpirun -np 4 ./ring_usempif08
+    - mpirun -np 4 ./connectivity_c
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    expire_in: 1 week
+
+test:ibm:
+  stage: test
+  tags: [darwin-slurm-shared]
+  variables:
+    SCHEDULER_PARAMETERS: "-ppower9 -t 2:00:00 -N 1 --ntasks-per-node=16"
+  dependencies:
+    - build:ibm
+  needs: ["build:ibm"]
+  script:
+    - pwd
+    - ls
+    - module load ibm
+    - export PATH=$PWD/install_test/bin:$PATH
+    - which mpirun
+    - pushd examples
+    - mpirun -np 4 hostname
+    - mpirun -np 4 ./hello_c
+    - mpirun -np 4 ./ring_c
+    - mpirun -np 4 ./hello_mpifh
+    - mpirun -np 4 ./ring_mpifh
+    - mpirun -np 4 ./hello_usempi
+    - mpirun -np 4 ./ring_usempi
+    - mpirun -np 4 ./hello_usempif08
+    - mpirun -np 4 ./ring_usempif08
+    - mpirun -np 4 ./connectivity_c
+    - popd
+    - mkdir osu-tests
+    - pushd osu-tests
+    - cp -p -r $OSU_TESTS_FOLDER/* .
+    - ./configure CC=mpicc FC=mpifort F77=mpifort CXX=mpiCC && make -j 8 clean && make -j 8 
+    - pushd mpi/pt2pt
+    - mpirun -np 2 ./osu_latency
+    - mpirun -np 2 ./osu_latency D H
+    - mpirun -np 2 ./osu_latency H D
+    - mpirun -np 2 ./osu_latency H H
+    - mpirun -np 2 ./osu_bw
+    - mpirun -np 2 ./osu_bw D H
+    - mpirun -np 2 ./osu_bw H D
+    - mpirun -np 2 ./osu_bw H H
+    - mpirun -np 2 ./osu_bibw
+    - mpirun -np 2 ./osu_bibw D H
+    - mpirun -np 2 ./osu_bibw H D
+    - mpirun -np 2 ./osu_bibw H H
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    expire_in: 1 week
+
+test:amd:
+  stage: test
+  tags: [darwin-slurm-shared]
+  variables:
+    SCHEDULER_PARAMETERS: "-pamd-rome -t 2:00:00 -N 1 --ntasks-per-node=16"
+  dependencies:
+    - build:amd
+  needs: ["build:amd"]
+  script:
+    - pwd
+    - ls
+    - module load aocc/3.0.0
+    - export PATH=$PWD/install_test/bin:$PATH
+    - export LD_LIBRARY_PATH=$PWD/install_test/lib:$LD_LIBRARY_PATH
+    - which mpirun
+    - cd examples
+    - mpirun -np 4 hostname
+    - mpirun -np 4 ./hello_c
+    - mpirun -np 4 ./ring_c
+    - mpirun -np 4 ./hello_mpifh
+    - mpirun -np 4 ./ring_mpifh
+    - mpirun -np 4 ./hello_usempi
+    - mpirun -np 4 ./ring_usempi
+    - mpirun -np 4 ./hello_usempif08
+    - mpirun -np 4 ./ring_usempif08
+    - mpirun -np 4 ./connectivity_c
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    expire_in: 1 week
+
+test:gnu:
+  stage: test
+  tags: [darwin-slurm-shared]
+  dependencies:
+    - build:gnu
+  needs: ["build:gnu"]
+  script:
+    - pwd
+    - ls
+    - module load gcc
+    - export PATH=$PWD/install_test/bin:$PATH
+    - which mpirun
+    - cd examples
+    - mpirun -np 4 hostname
+    - mpirun -np 4 ./hello_c
+    - mpirun -np 4 ./ring_c
+    - mpirun -np 4 ./hello_mpifh
+    - mpirun -np 4 ./ring_mpifh
+    - mpirun -np 4 ./hello_usempi
+    - mpirun -np 4 ./ring_usempi
+    - mpirun -np 4 ./hello_usempif08
+    - mpirun -np 4 ./ring_usempif08
+    - mpirun -np 4 ./connectivity_c
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    expire_in: 1 week
+
+test:cce:
+  stage: test
+  tags: [darwin-slurm-shared]
+  variables:
+    SCHEDULER_PARAMETERS: "-pcrossroads-dev -t 4:00:00 -N 1 --ntasks-per-node=16"
+  dependencies:
+    - build:cce
+  needs: ["build:cce"]
+  script:
+    - pwd
+    - ls
+    - hostname
+    - module use --append /opt/cray/pe/modulefiles
+    - module avail cce
+    - module load cce
+    - module unload libsci
+    - module unload cray-mvapich2_nogpu
+    - export PATH=$PWD/install_test/bin:$PATH
+    - which mpirun
+    - cd examples
+    - mpirun -np 4 hostname
+    - mpirun -np 4 ./hello_c
+    - mpirun -np 4 ./ring_c
+    - mpirun -np 4 ./hello_mpifh
+    - mpirun -np 4 ./ring_mpifh
+    - mpirun -np 4 ./hello_usempi
+    - mpirun -np 4 ./ring_usempi
+    - mpirun -np 4 ./hello_usempif08
+    - mpirun -np 4 ./ring_usempif08
+    - mpirun -np 4 ./connectivity_c
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    expire_in: 1 week
+

.ci/mellanox/README.md

@@ -0,0 +1,16 @@
+# Open MPI Continuous Integration (CI) Services
+## Mellanox Open MPI CI
+### Scope
+[Mellanox](https://www.mellanox.com/) Open MPI CI is intended to verify Open MPI with recent Mellanox SW components ([Mellanox OFED](https://www.mellanox.com/page/products_dyn?product_family=26), [UCX](https://www.mellanox.com/page/products_dyn?product_family=281&mtag=ucx) and other [HPC-X](https://www.mellanox.com/page/products_dyn?product_family=189&mtag=hpc-x) components) in the Mellanox lab environment.
+
+CI is managed by [Azure Pipelines](https://docs.microsoft.com/en-us/azure/devops/pipelines/?view=azure-devops) service.
+
+Mellanox Open MPI CI includes:
+* Open MPI building with internal stable engineering versions of UCX and HCOLL. The building is run in Docker-based environment.
+* Sanity functional testing.
+### How to Run CI
+Mellanox Open MPI CI is triggered upon the following events:
+* Create a pull request (PR). CI status is visible in the PR status. CI is restarted automatically upon each new commit within the PR. CI status and log files are also available on the Azure DevOps server.
+* Trigger CI with special PR comments (for example, `/azp run`). Comment triggers are available only if the comment author has write permission to the PR target repo. Detailed information about comment triggers is available in the official Azure DevOps [documentation](https://docs.microsoft.com/en-us/azure/devops/pipelines/repos/github?view=azure-devops&tabs=yaml#comment-triggers).
+### Support
+In case of any issues, questions or suggestions please contact to [Mellanox Open MPI CI support team](mailto:[email protected]).

.ci/mellanox/azure-pipelines.yml

@@ -1,6 +1,6 @@
 trigger: none
 pr:
-  - master
+  - main
   - v*.*.x
 
 pool: