From ed2250a3440097cb1a3db75b28b993e8f8c453e9 Mon Sep 17 00:00:00 2001 From: Pramod Kumbhar Date: Thu, 17 Dec 2020 07:31:43 +0100 Subject: [PATCH 01/17] CMake build options refactoring for Makefile * move all makefile related options into single file MakefileBuildOptions.cmake * remove all option processing from extra/CMakeLists.txt * fix with neuron linking : type CN_PGI_ACC_FLAGS -> PGI_ACC_FLAGS --- CMake/MakefileBuildOptions.cmake | 73 ++++++++++++++++++++++++++++++++ CMake/OpenAccHelper.cmake | 51 +++++++++++++--------- CMakeLists.txt | 32 ++++---------- coreneuron/CMakeLists.txt | 11 ----- coreneuron/apps/main1.cpp | 6 +-- extra/CMakeLists.txt | 64 ++-------------------------- extra/nrnivmodl_core_makefile.in | 30 +++++++++---- 7 files changed, 140 insertions(+), 127 deletions(-) create mode 100644 CMake/MakefileBuildOptions.cmake diff --git a/CMake/MakefileBuildOptions.cmake b/CMake/MakefileBuildOptions.cmake new file mode 100644 index 000000000..35cb42832 --- /dev/null +++ b/CMake/MakefileBuildOptions.cmake @@ -0,0 +1,73 @@ +# ============================================================================= +# Common CXX and ISPC flags +# ============================================================================= + +# ISPC should compile with --pic by default +set(CMAKE_ISPC_FLAGS "${CMAKE_ISPC_FLAGS} --pic") + +# ============================================================================= +# NMODL CLI options : common and backend specific +# ============================================================================= +# if user pass arguments then use those as common arguments +if ("${CORENRN_NMODL_FLAGS}" STREQUAL "") + set(NMODL_COMMON_ARGS "passes --inline") +else() + set(NMODL_COMMON_ARGS ${CORENRN_NMODL_FLAGS}) +endif() + +set(NMODL_CPU_BACKEND_ARGS "host --c") +set(NMODL_ISPC_BACKEND_ARGS "host --ispc") +set(NMODL_ACC_BACKEND_ARGS "host --c acc --oacc") + +# ============================================================================= +# Extract Compile definitions : common to all backend +# ============================================================================= +get_directory_property(COMPILE_DEFS COMPILE_DEFINITIONS) +if(COMPILE_DEFS) + set(CORENRN_COMMON_COMPILE_DEFS "") + foreach(flag ${COMPILE_DEFS}) + set(CORENRN_COMMON_COMPILE_DEFS "${CORENRN_COMMON_COMPILE_DEFS} -D${flag}") + endforeach() +endif() + +# ============================================================================= +# link flags : common to all backend +# ============================================================================= +# ~~~ +# find_cuda uses FindThreads that adds below imported target we +# shouldn't add imported target to link line +# ~~~ +list(REMOVE_ITEM CORENRN_LINK_LIBS "Threads::Threads") + +# replicate CMake magic to transform system libs to -l +foreach(link_lib ${CORENRN_LINK_LIBS}) + if(${link_lib} MATCHES "\-l.*") + string(APPEND CORENRN_COMMON_LDFLAGS " ${link_lib}") + continue() + endif() + get_filename_component(path ${link_lib} DIRECTORY) + if(NOT path) + string(APPEND CORENRN_COMMON_LDFLAGS " -l${link_lib}") + elseif("${path}" MATCHES "^(/lib|/lib64|/usr/lib|/usr/lib64)$") + get_filename_component(libname ${link_lib} NAME_WE) + string(REGEX REPLACE "^lib" "" libname ${libname}) + string(APPEND CORENRN_COMMON_LDFLAGS " -l${libname}") + else() + string(APPEND CORENRN_COMMON_LDFLAGS " ${link_lib}") + endif() +endforeach() + +# ============================================================================= +# compile flags : common to all backend +# ============================================================================= +# PGI compiler adds --c++14;-A option for C++14, remove ";" +string(REPLACE ";" " " CXX14_STD_FLAGS "${CMAKE_CXX14_STANDARD_COMPILE_OPTION}") +string(TOUPPER "${CMAKE_BUILD_TYPE}" _BUILD_TYPE) +set(CORENRN_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${_BUILD_TYPE}} ${CXX14_STD_FLAGS}") + +# ============================================================================= +# nmodl/mod2c related options : TODO +# ============================================================================= +# name of nmodl/mod2c binary +get_filename_component(nmodl_name ${CORENRN_MOD2CPP_BINARY} NAME) +set(nmodl_binary_name ${nmodl_name}) diff --git a/CMake/OpenAccHelper.cmake b/CMake/OpenAccHelper.cmake index 1d7ce7c2c..eab63e710 100644 --- a/CMake/OpenAccHelper.cmake +++ b/CMake/OpenAccHelper.cmake @@ -4,10 +4,14 @@ # See top-level LICENSE file for details. # ============================================================================= +# ============================================================================= +# Prepare compiler flags for GPU target +# ============================================================================= if(CORENRN_ENABLE_GPU) + # cuda unified memory support if(CORENRN_ENABLE_CUDA_UNIFIED_MEMORY) - add_definitions(-DUNIFIED_MEMORY) + set(UNIFIED_MEMORY_DEF -DUNIFIED_MEMORY) endif() # if user don't specify host compiler, use gcc from $PATH @@ -20,32 +24,24 @@ if(CORENRN_ENABLE_GPU) # various flags for PGI compiler with GPU build if(${CMAKE_C_COMPILER_ID} STREQUAL "PGI") + # workaround for old PGI version - add_definitions(-DPG_ACC_BUGS) - set(ACC_FLAGS "-acc") + set(PGI_ACC_BUG_DEFS -DPG_ACC_BUGS) + set(PGI_ACC_FLAGS "-acc") # disable very verbose diagnosis messages and obvious warnings for mod2c set(PGI_DIAG_FLAGS "--diag_suppress 161,177,550") - # some of the mod files can have too many functions, increase inline level + # inlining of large functions for OpenACC set(PGI_INLINE_FLAGS "-Minline=size:200,levels:10") - # C/C++ compiler flags - set(CMAKE_C_FLAGS "${ACC_FLAGS} ${CMAKE_C_FLAGS}") - set(CMAKE_CXX_FLAGS "${ACC_FLAGS} ${CMAKE_CXX_FLAGS} ${PGI_DIAG_FLAGS}") + # avoid PGI adding standard compliant "-A" flags set(CMAKE_CXX11_STANDARD_COMPILE_OPTION --c++11) set(CMAKE_CXX14_STANDARD_COMPILE_OPTION --c++14) + else() message(FATAL_ERROR "GPU support is available via OpenACC using PGI/NVIDIA compilers." " Use NVIDIA HPC SDK with -DCMAKE_C_COMPILER=nvc -DCMAKE_CXX_COMPILER=nvc++") endif() - # set property for neuron to link with coreneuron libraries - set_property( - GLOBAL - PROPERTY - CORENEURON_LIB_LINK_FLAGS - "-acc -rdynamic -lrt -Wl,--whole-archive -L${CMAKE_HOST_SYSTEM_PROCESSOR} -lcorenrnmech -L${CMAKE_INSTALL_PREFIX}/lib -lcoreneuron -lcudacoreneuron -Wl,--no-whole-archive ${CUDA_cudart_static_LIBRARY}" - ) - # find_cuda produce verbose messages : use new behavior to use _ROOT variables if(POLICY CMP0074) cmake_policy(SET CMP0074 NEW) @@ -53,11 +49,26 @@ if(CORENRN_ENABLE_GPU) find_package(CUDA 9.0 REQUIRED) set(CUDA_SEPARABLE_COMPILATION ON) set(CUDA_PROPAGATE_HOST_FLAGS OFF) - add_definitions(-DCUDA_PROFILING) -else(CORENRN_ENABLE_GPU) - # OpenACC pragmas are not guarded, disable all unknown pragm warnings - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${IGNORE_UNKNOWN_PRAGMA_FLAGS}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${IGNORE_UNKNOWN_PRAGMA_FLAGS}") + set(CUDA_PROFILING_DEF -DCUDA_PROFILING) + + set(CORENRN_ACC_GPU_DEFS "${UNIFIED_MEMORY_DEF} ${CUDA_PROFILING_DEF} ${PGI_ACC_BUG_DEFS}") + set(CORENRN_ACC_GPU_FLAGS "${PGI_ACC_FLAGS} ${PGI_DIAG_FLAGS} ${PGI_INLINE_FLAGS}") + + add_definitions(${CORENRN_ACC_GPU_DEFS}) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CORENRN_ACC_GPU_FLAGS}") +endif() + +# ============================================================================= +# Set global property that will be used by NEURON to link with CoreNEURON +# ============================================================================= +if(CORENRN_ENABLE_GPU) + set_property( + GLOBAL + PROPERTY + CORENEURON_LIB_LINK_FLAGS + "${PGI_ACC_FLAGS} -rdynamic -lrt -Wl,--whole-archive -L${CMAKE_HOST_SYSTEM_PROCESSOR} -lcorenrnmech -L${CMAKE_INSTALL_PREFIX}/lib -lcoreneuron -lcudacoreneuron -Wl,--no-whole-archive ${CUDA_cudart_static_LIBRARY}" + ) +else() set_property(GLOBAL PROPERTY CORENEURON_LIB_LINK_FLAGS "-L${CMAKE_HOST_SYSTEM_PROCESSOR} -lcorenrnmech") endif(CORENRN_ENABLE_GPU) diff --git a/CMakeLists.txt b/CMakeLists.txt index 811605c3c..99a79dc03 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -95,13 +95,6 @@ set(LIKWID_DIR "" CACHE PATH "Path to likwid performance analysis suite") -set(CORENRN_FRONTEND_C_COMPILER - gcc - CACHE FILEPATH "C compiler for building mod2c [frontend]") -set(CORENRN_FRONTEND_CXX_COMPILER - g++ - CACHE FILEPATH "C++ compiler for building mod2c [frontend]") - if(CORENEURON_AS_SUBPROJECT) set(CORENRN_ENABLE_UNIT_TESTS OFF) endif() @@ -126,11 +119,6 @@ include(OpenAccHelper) find_package(PythonInterp REQUIRED) find_package(Perl REQUIRED) -# ============================================================================= -# ISPC should compile with --pic by default -# ============================================================================= -set(CMAKE_ISPC_FLAGS "--pic ${CMAKE_ISPC_FLAGS}") - # ============================================================================= # Common build options # ============================================================================= @@ -147,7 +135,6 @@ endif() # Build option specific compiler flags # ============================================================================= if(${CMAKE_CXX_COMPILER_ID} STREQUAL "PGI") - add_definitions(-DSWAP_ENDIAN_DISABLE_ASM) # PGI with llvm code generation doesn't have necessary assembly intrinsic headers add_definitions(-DEIGEN_DONT_VECTORIZE=1) endif() @@ -168,7 +155,6 @@ endif() if(CORENRN_ENABLE_ISPC) enable_language(ISPC) - add_definitions("-DISPC_INTEROP=1") set(CORENRN_ENABLE_NMODL ON) endif() @@ -283,7 +269,6 @@ if(CORENRN_ENABLE_NMODL) if(CORENRN_ENABLE_GPU) string(APPEND CORENRN_NMODL_FLAGS " acc --oacc") endif() - separate_arguments(NMODL_EXTRA_FLAGS_LIST UNIX_COMMAND "${CORENRN_NMODL_FLAGS}") else() include(AddMod2cSubmodule) set(CORENRN_MOD2CPP_BINARY ${CMAKE_BINARY_DIR}/bin/mod2c_core${CMAKE_EXECUTABLE_SUFFIX}) @@ -309,10 +294,17 @@ if(CORENRN_ENABLE_LIKWID_PROFILING) add_definitions("-DLIKWID_PERFMON") endif() +# ============================================================================= +# Common CXX flags : ignore unknown pragma warnings +# ============================================================================= +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${IGNORE_UNKNOWN_PRAGMA_FLAGS}") + # ============================================================================= # Add main directories # ============================================================================= add_subdirectory(coreneuron) + +include(MakefileBuildOptions) add_subdirectory(extra) if(CORENRN_ENABLE_UNIT_TESTS) @@ -325,14 +317,6 @@ endif() install(FILES CMake/coreneuron-config.cmake DESTINATION share/cmake) install(EXPORT coreneuron DESTINATION share/cmake) -# just for printing the compiler flags in the build status -string(TOUPPER ${CMAKE_BUILD_TYPE} BUILD_TYPE_UPPER) -if(BUILD_TYPE_UPPER MATCHES "CUSTOM") - set(COMPILER_FLAGS "${CMAKE_CXX_FLAGS}") -else() - set(COMPILER_FLAGS "${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}}") -endif() - if(NOT CORENEURON_AS_SUBPROJECT) # ============================================================================= # Setup Doxygen documentation @@ -402,7 +386,7 @@ if(cmake_generator_tolower MATCHES "makefile") message(STATUS "C COMPILER | ${CMAKE_C_COMPILER}") message(STATUS "CXX COMPILER | ${CMAKE_CXX_COMPILER}") - message(STATUS "COMPILE FLAGS | ${COMPILER_FLAGS} ${CMAKE_CXX_FLAGS}") + message(STATUS "COMPILE FLAGS | ${CORENRN_CXX_FLAGS}") message(STATUS "Build Type | ${COMPILE_LIBRARY_TYPE}") message(STATUS "MPI | ${CORENRN_ENABLE_MPI}") if(CORENRN_ENABLE_MPI) diff --git a/coreneuron/CMakeLists.txt b/coreneuron/CMakeLists.txt index fff6b4549..c45c1b4e1 100644 --- a/coreneuron/CMakeLists.txt +++ b/coreneuron/CMakeLists.txt @@ -56,11 +56,6 @@ set(NMODL_UNITS_FILE "${CMAKE_BINARY_DIR}/share/mod2c/nrnunits.lib") file(COPY ${CORENEURON_PROJECT_SOURCE_DIR}/coreneuron/mechanism/mech/modfile DESTINATION ${CMAKE_BINARY_DIR}/share) -# eion.cpp depends on CORENRN_USE_LEGACY_UNITS -set(LegacyFR_FILES mechanism/eion.cpp apps/main1.cpp io/global_vars.cpp) -set_source_files_properties(${LegacyFR_FILES} PROPERTIES COMPILE_FLAGS - "-DCORENRN_USE_LEGACY_UNITS=${CORENRN_USE_LEGACY_UNITS}") - # ============================================================================= # coreneuron GPU library # ============================================================================= @@ -85,12 +80,6 @@ if(CORENRN_ENABLE_GPU) set_source_files_properties(${OPENACC_EXCLUDED_FILES} PROPERTIES COMPILE_FLAGS "-DDISABLE_OPENACC") - # TODO : only older PGI versions? - if(${CMAKE_C_COMPILER_ID} STREQUAL "PGI") - set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/scopmath_core/sparse_thread.c - PROPERTIES COMPILE_FLAGS "-ta=tesla:nollvm") - endif() - # compile cuda files for multiple architecture cuda_add_library( "cudacoreneuron" ${CORENEURON_CUDA_FILES} diff --git a/coreneuron/apps/main1.cpp b/coreneuron/apps/main1.cpp index 0c25b65e0..2ef86b6df 100644 --- a/coreneuron/apps/main1.cpp +++ b/coreneuron/apps/main1.cpp @@ -77,10 +77,8 @@ bool corenrn_units_use_legacy() { void (*nrn2core_part2_clean_)(); -#ifdef ISPC_INTEROP // cf. utils/ispc_globals.c extern double ispc_celsius; -#endif /** * If "export OMP_NUM_THREADS=n" is not set then omp by default sets @@ -213,9 +211,9 @@ void nrn_init_and_load_data(int argc, corenrn_param.celsius = celsius; -#ifdef ISPC_INTEROP + // for ispc backend ispc_celsius = celsius; -#endif + // create net_cvode instance mk_netcvode(); diff --git a/extra/CMakeLists.txt b/extra/CMakeLists.txt index a7cf482fa..7036f7035 100644 --- a/extra/CMakeLists.txt +++ b/extra/CMakeLists.txt @@ -5,71 +5,15 @@ # ============================================================================= # ============================================================================= -# Prepare nrnivmodl-core script with correct compiler flags +# Copy first into build directory as it will be used for special-core # ============================================================================= - -# extract the COMPILE_DEFINITIONS property from the directory. -get_directory_property(CORENRN_COMPILE_FLAGS COMPILE_DEFINITIONS) -if(CORENRN_COMPILE_FLAGS) - set(CORENRN_COMPILE_DEFS "") - foreach(flag ${CORENRN_COMPILE_FLAGS}) - set(CORENRN_COMPILE_DEFS "${CORENRN_COMPILE_DEFS} -D${flag}") - endforeach() -endif() - -# ~~~ -# find_cuda uses FindThreads that adds below imported target we shouldn't add -# imported target to link line -# ~~~ -list(REMOVE_ITEM CORENRN_LINK_LIBS "Threads::Threads") -# replicate CMake magic to transform system libs to -l -foreach(link_lib ${CORENRN_LINK_LIBS}) - if(${link_lib} MATCHES "\-l.*") - string(APPEND CORENRN_LINK_DEFS " ${link_lib}") - continue() - endif() - get_filename_component(path ${link_lib} DIRECTORY) - if(NOT path) - string(APPEND CORENRN_LINK_DEFS " -l${link_lib}") - elseif("${path}" MATCHES "^(/lib|/lib64|/usr/lib|/usr/lib64)$") - get_filename_component(libname ${link_lib} NAME_WE) - string(REGEX REPLACE "^lib" "" libname ${libname}) - string(APPEND CORENRN_LINK_DEFS " -l${libname}") - else() - string(APPEND CORENRN_LINK_DEFS " ${link_lib}") - endif() -endforeach() - -# PGI compiler add --c++14;-A option for c++14 flag -string(REPLACE ";" " " CXX14_STANDARD_COMPILE_OPTION "${CMAKE_CXX14_STANDARD_COMPILE_OPTION}") - -# compiler flags depending on BUILD_TYPE (configured as BUILD_TYPE__FLAGS) -string(TOUPPER "${CMAKE_BUILD_TYPE}" _BUILD_TYPE) -set(BUILD_TYPE_C_FLAGS "${CMAKE_C_FLAGS_${_BUILD_TYPE}}") -set(BUILD_TYPE_CXX_FLAGS "${CMAKE_CXX_FLAGS_${_BUILD_TYPE}}") -message(STATUS "CXX Compile flags from BUILD_TYPE (${_BUILD_TYPE}): ${BUILD_TYPE_CXX_FLAGS}") - -# nmodl options -if(CORENRN_ENABLE_NMODL) - set(nmodl_arguments_c "host --c passes --inline ${CORENRN_NMODL_FLAGS}") - set(nmodl_arguments_ispc "host --ispc passes --inline ${CORENRN_NMODL_FLAGS}") -else() - set(nmodl_arguments_c "") - set(nmodl_arguments_ispc "") -endif() - -# name of nmodl/mod2c binary -get_filename_component(nmodl_name ${CORENRN_MOD2CPP_BINARY} NAME) -set(nmodl_binary_name ${nmodl_name}) - -# ============================================================================= -# Install first into build directory and then to install prefix -# ============================================================================= - configure_file(nrnivmodl_core_makefile.in ${CMAKE_BINARY_DIR}/share/coreneuron/nrnivmodl_core_makefile @ONLY) configure_file(nrnivmodl-core.in ${CMAKE_BINARY_DIR}/bin/nrnivmodl-core @ONLY) +# ============================================================================= +# Install for end users +# ============================================================================= install(FILES ${CMAKE_BINARY_DIR}/share/coreneuron/nrnivmodl_core_makefile DESTINATION share/coreneuron) install(PROGRAMS ${CMAKE_BINARY_DIR}/bin/nrnivmodl-core DESTINATION bin) diff --git a/extra/nrnivmodl_core_makefile.in b/extra/nrnivmodl_core_makefile.in index fda9c7e41..89be1c8e2 100644 --- a/extra/nrnivmodl_core_makefile.in +++ b/extra/nrnivmodl_core_makefile.in @@ -35,7 +35,7 @@ MOD_TO_CPP_DIR = $(OUTPUT_DIR)/corenrn/mod2c MOD_OBJS_DIR = $(OUTPUT_DIR)/corenrn/build # Linked libraries gathered by CMake -LDFLAGS = $(LINKFLAGS) @CORENRN_LINK_DEFS@ +LDFLAGS = $(LINKFLAGS) @CORENRN_COMMON_LDFLAGS@ CORENRNLIB_FLAGS = -L$(CORENRN_LIB_DIR) -lcoreneuron CORENRNLIB_FLAGS += $(if @reportinglib_LIB_DIR@, -W$(subst ;, -W,l,-rpath,@reportinglib_LIB_DIR@),) CORENRNLIB_FLAGS += $(if @sonatareport_LIB_DIR@, -W$(subst ;, -W,l,-rpath,@sonatareport_LIB_DIR@),) @@ -49,8 +49,8 @@ INCLUDES += $(if @reportinglib_INCLUDE_DIR@, -I$(subst ;, -I,@reportinglib_INCLU # C++ compilation and link commands CXX = @CMAKE_CXX_COMPILER@ -CXXFLAGS = @BUILD_TYPE_CXX_FLAGS@ @CMAKE_CXX_FLAGS@ @CXX14_STANDARD_COMPILE_OPTION@ @PGI_INLINE_FLAGS@ -CXX_COMPILE_CMD = $(CXX) $(CXXFLAGS) @CMAKE_CXX_COMPILE_OPTIONS_PIC@ @CORENRN_COMPILE_DEFS@ $(INCLUDES) +CXXFLAGS = @CORENRN_CXX_FLAGS@ +CXX_COMPILE_CMD = $(CXX) $(CXXFLAGS) @CMAKE_CXX_COMPILE_OPTIONS_PIC@ @CORENRN_COMMON_COMPILE_DEFS@ $(INCLUDES) CXX_LINK_EXE_CMD = $(CXX) $(CXXFLAGS) @CMAKE_EXE_LINKER_FLAGS@ CXX_SHARED_LIB_CMD = $(CXX) $(CXXFLAGS) @CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS@ @CMAKE_SHARED_LIBRARY_CXX_FLAGS@ @CMAKE_SHARED_LINKER_FLAGS@ @@ -61,6 +61,16 @@ ISPC_COMPILE_CMD = $(ISPC) @CMAKE_ISPC_FLAGS@ -I$(CORENRN_INC_DIR) # env variables required for mod2c or nmodl MOD2CPP_ENV_VAR = PYTHONPATH=@CORENRN_NMODL_PYTHONPATH@:${CORENRN_LIB_DIR}/python MODLUNIT=$(CORENRN_SHARE_MOD2CPP_DIR)/nrnunits.lib +# nmodl options +ifeq (@CORENRN_ENABLE_NMODL@, ON) + ifeq (@CORENRN_ENABLE_GPU@, ON) + nmodl_arguments_c="@NMODL_ACC_BACKEND_ARGS@ @NMODL_COMMON_ARGS@" + else + nmodl_arguments_c="@NMODL_CPU_BACKEND_ARGS@ @NMODL_COMMON_ARGS@" + endif + nmodl_arguments_ispc="@NMODL_ISPC_BACKEND_ARGS@ @NMODL_COMMON_ARGS@" +endif + # name of the mechanism library with suffix if provided COREMECH_LIB_NAME = corenrnmech$(if $(MECHLIB_SUFFIX),_$(MECHLIB_SUFFIX),) COREMECH_LIB_PATH = $(OUTPUT_DIR)/lib$(COREMECH_LIB_NAME)$(LIB_SUFFIX) @@ -144,13 +154,17 @@ C_RESET := \033[0m C_GREEN := \033[32m # Default nmodl flags. Override if MOD2CPP_RUNTIME_FLAGS is not empty -NMODL_FLAGS_ISPC = $(if $(MOD2CPP_RUNTIME_FLAGS),$(MOD2CPP_RUNTIME_FLAGS),@nmodl_arguments_ispc@) -MOD2CPP_FLAGS_C = $(if $(MOD2CPP_RUNTIME_FLAGS),$(MOD2CPP_RUNTIME_FLAGS),@nmodl_arguments_c@) +ifeq (@CORENRN_ENABLE_NMODL@, ON) + MOD2CPP_FLAGS_ISPC = $(if $(MOD2CPP_RUNTIME_FLAGS),$(MOD2CPP_RUNTIME_FLAGS),@nmodl_arguments_ispc@) + MOD2CPP_FLAGS_C = $(if $(MOD2CPP_RUNTIME_FLAGS),$(MOD2CPP_RUNTIME_FLAGS),@nmodl_arguments_c@) +endif + ifeq (@CORENRN_ENABLE_ISPC@, ON) - $(info Default nmodl flags: @nmodl_arguments_ispc@) + $(info Default NMODL flags: @nmodl_arguments_ispc@) else - $(info Default nmodl flags: @nmodl_arguments_c@) + $(info Default NMODL flags: @nmodl_arguments_c@) endif + ifneq ($(MOD2CPP_RUNTIME_FLAGS),) $(warning Runtime nmodl flags (they replace the default ones): $(MOD2CPP_RUNTIME_FLAGS)) endif @@ -198,7 +212,7 @@ $(MOD_OBJS_DIR)/%.obj: $(MOD_TO_CPP_DIR)/%.ispc | $(MOD_OBJS_DIR) # translate MOD files to ISPC using NMODL $(mod_ispc_files): $(MOD_TO_CPP_DIR)/%.ispc: $(MODS_PATH)/%.mod | $(MOD_TO_CPP_DIR) - $(MOD2CPP_ENV_VAR) $(MOD2CPP_BINARY_PATH) $< -o $(MOD_TO_CPP_DIR)/ $(NMODL_FLAGS_ISPC) + $(MOD2CPP_ENV_VAR) $(MOD2CPP_BINARY_PATH) $< -o $(MOD_TO_CPP_DIR)/ $(MOD2CPP_FLAGS_ISPC) # translate MOD files to CPP using mod2c/NMODL $(mod_cpp_files): $(MOD_TO_CPP_DIR)/%.cpp: $(MODS_PATH)/%.mod | $(MOD_TO_CPP_DIR) From 31e116d25ed4bc872184b71c2b21e200b25d593a Mon Sep 17 00:00:00 2001 From: Pramod Kumbhar Date: Tue, 5 Jan 2021 15:25:30 +0100 Subject: [PATCH 02/17] Change to latest NVHPC --- tests/jenkins/install_coreneuron.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/jenkins/install_coreneuron.sh b/tests/jenkins/install_coreneuron.sh index 9dc30384b..761245eec 100755 --- a/tests/jenkins/install_coreneuron.sh +++ b/tests/jenkins/install_coreneuron.sh @@ -10,7 +10,7 @@ reportinglib_dir=$(spack location --install-dir --latest reportinglib%intel) CORENRN_TYPE="$1" if [ "${CORENRN_TYPE}" = "GPU-non-unified" ] || [ "${CORENRN_TYPE}" = "GPU-unified" ]; then - module load pgi/19.10 cuda hpe-mpi cmake boost + module load nvhpc/20.11 cuda/11.1.0 hpe-mpi cmake boost mkdir build_${CORENRN_TYPE} else From ddc7ebc912e153a32af9994885120c8ffcb1aa2a Mon Sep 17 00:00:00 2001 From: Pramod S Kumbhar Date: Tue, 5 Jan 2021 22:44:39 +0100 Subject: [PATCH 03/17] fix ISPC build issue and nrnivmodl-core failure on OSX --- extra/nrnivmodl_core_makefile.in | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/extra/nrnivmodl_core_makefile.in b/extra/nrnivmodl_core_makefile.in index 89be1c8e2..49149d614 100644 --- a/extra/nrnivmodl_core_makefile.in +++ b/extra/nrnivmodl_core_makefile.in @@ -15,6 +15,12 @@ OUTPUT_DIR = @CMAKE_HOST_SYSTEM_PROCESSOR@ DESTDIR = TARGET_LIB_TYPE = $(BUILD_TYPE) +# required for OSX to execute nrnivmodl-core +OSX_SYSROOT=@CMAKE_OSX_SYSROOT@ +ifneq ($(OSX_SYSROOT),) + export SDKROOT := $(OSX_SYSROOT) +endif + # CoreNEURON installation directories CORENRN_BIN_DIR := $(ROOT)/bin CORENRN_LIB_DIR := $(ROOT)/lib @@ -64,11 +70,11 @@ MOD2CPP_ENV_VAR = PYTHONPATH=@CORENRN_NMODL_PYTHONPATH@:${CORENRN_LIB_DIR}/pytho # nmodl options ifeq (@CORENRN_ENABLE_NMODL@, ON) ifeq (@CORENRN_ENABLE_GPU@, ON) - nmodl_arguments_c="@NMODL_ACC_BACKEND_ARGS@ @NMODL_COMMON_ARGS@" + nmodl_arguments_c=@NMODL_ACC_BACKEND_ARGS@ @NMODL_COMMON_ARGS@ else - nmodl_arguments_c="@NMODL_CPU_BACKEND_ARGS@ @NMODL_COMMON_ARGS@" + nmodl_arguments_c=@NMODL_CPU_BACKEND_ARGS@ @NMODL_COMMON_ARGS@ endif - nmodl_arguments_ispc="@NMODL_ISPC_BACKEND_ARGS@ @NMODL_COMMON_ARGS@" + nmodl_arguments_ispc=@NMODL_ISPC_BACKEND_ARGS@ @NMODL_COMMON_ARGS@ endif # name of the mechanism library with suffix if provided @@ -155,8 +161,8 @@ C_GREEN := \033[32m # Default nmodl flags. Override if MOD2CPP_RUNTIME_FLAGS is not empty ifeq (@CORENRN_ENABLE_NMODL@, ON) - MOD2CPP_FLAGS_ISPC = $(if $(MOD2CPP_RUNTIME_FLAGS),$(MOD2CPP_RUNTIME_FLAGS),@nmodl_arguments_ispc@) - MOD2CPP_FLAGS_C = $(if $(MOD2CPP_RUNTIME_FLAGS),$(MOD2CPP_RUNTIME_FLAGS),@nmodl_arguments_c@) + MOD2CPP_FLAGS_ISPC = $(if $(MOD2CPP_RUNTIME_FLAGS),$(MOD2CPP_RUNTIME_FLAGS),$(nmodl_arguments_ispc)) + MOD2CPP_FLAGS_C = $(if $(MOD2CPP_RUNTIME_FLAGS),$(MOD2CPP_RUNTIME_FLAGS),$(nmodl_arguments_c)) endif ifeq (@CORENRN_ENABLE_ISPC@, ON) From 4bbe8ef0d1644aba7a0a727fbda5169e5003242a Mon Sep 17 00:00:00 2001 From: Pramod Kumbhar Date: Tue, 5 Jan 2021 23:15:24 +0100 Subject: [PATCH 04/17] Avoid P2 UC2 ndoes as GPFS is not mounted (HELP-13390) --- tests/jenkins/install_coreneuron.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/jenkins/install_coreneuron.sh b/tests/jenkins/install_coreneuron.sh index 761245eec..23a0359b8 100755 --- a/tests/jenkins/install_coreneuron.sh +++ b/tests/jenkins/install_coreneuron.sh @@ -38,8 +38,8 @@ if [ "${CORENRN_TYPE}" = "GPU-non-unified" ]; then -DCORENRN_ENABLE_GPU=ON \ -DCORENRN_ENABLE_CUDA_UNIFIED_MEMORY=OFF \ -DCMAKE_INSTALL_PREFIX=$WORKSPACE/install_${CORENRN_TYPE}/ \ - -DTEST_MPI_EXEC_BIN="srun;--account=proj16;--partition=$SALLOC_PARTITION;--constraint=volta;--gres=gpu:2;--mem;0;-t;00:05:00" \ - -DTEST_EXEC_PREFIX="srun;-n;6;--account=proj16;--partition=$SALLOC_PARTITION;--constraint=volta;--gres=gpu:2;--mem;0;-t;00:05:00" \ + -DTEST_MPI_EXEC_BIN="srun;--account=proj16;--partition=$SALLOC_PARTITION;--constraint=volta,skl;--gres=gpu:2;--mem;0;-t;00:05:00" \ + -DTEST_EXEC_PREFIX="srun;-n;6;--account=proj16;--partition=$SALLOC_PARTITION;--constraint=volta,skl;--gres=gpu:2;--mem;0;-t;00:05:00" \ -DAUTO_TEST_WITH_SLURM=OFF \ -DAUTO_TEST_WITH_MPIEXEC=OFF \ $WORKSPACE/ @@ -48,8 +48,8 @@ elif [ "${CORENRN_TYPE}" = "GPU-unified" ]; then -DCORENRN_ENABLE_GPU=ON \ -DCORENRN_ENABLE_CUDA_UNIFIED_MEMORY=ON \ -DCMAKE_INSTALL_PREFIX=$WORKSPACE/install_${CORENRN_TYPE}/ \ - -DTEST_MPI_EXEC_BIN="srun;--account=proj16;--partition=$SALLOC_PARTITION;--constraint=volta;--gres=gpu:2;--mem;0;-t;00:05:00" \ - -DTEST_EXEC_PREFIX="srun;-n;6;--account=proj16;--partition=$SALLOC_PARTITION;--constraint=volta;--gres=gpu:2;--mem;0;-t;00:05:00" \ + -DTEST_MPI_EXEC_BIN="srun;--account=proj16;--partition=$SALLOC_PARTITION;--constraint=volta,skl;--gres=gpu:2;--mem;0;-t;00:05:00" \ + -DTEST_EXEC_PREFIX="srun;-n;6;--account=proj16;--partition=$SALLOC_PARTITION;--constraint=volta,skl;--gres=gpu:2;--mem;0;-t;00:05:00" \ -DAUTO_TEST_WITH_SLURM=OFF \ -DAUTO_TEST_WITH_MPIEXEC=OFF \ $WORKSPACE/ From 71c4a5754ad05cfb2a30494a90ee411b22d982fd Mon Sep 17 00:00:00 2001 From: Pramod Kumbhar Date: Wed, 6 Jan 2021 00:30:55 +0100 Subject: [PATCH 05/17] Issue with unified memory and cuda driver compatibility - downgrade modules --- tests/jenkins/install_coreneuron.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/jenkins/install_coreneuron.sh b/tests/jenkins/install_coreneuron.sh index 23a0359b8..c2f3b614d 100755 --- a/tests/jenkins/install_coreneuron.sh +++ b/tests/jenkins/install_coreneuron.sh @@ -10,7 +10,7 @@ reportinglib_dir=$(spack location --install-dir --latest reportinglib%intel) CORENRN_TYPE="$1" if [ "${CORENRN_TYPE}" = "GPU-non-unified" ] || [ "${CORENRN_TYPE}" = "GPU-unified" ]; then - module load nvhpc/20.11 cuda/11.1.0 hpe-mpi cmake boost + module load nvhpc/20.9 cuda/10.1.243 hpe-mpi cmake boost mkdir build_${CORENRN_TYPE} else From 316c12693619e087bf371aa53ac3cd82ab1a54f7 Mon Sep 17 00:00:00 2001 From: Pramod Kumbhar Date: Wed, 6 Jan 2021 07:12:10 +0100 Subject: [PATCH 06/17] Issue with PGI compiler due to makelocalrc on jenkins --- tests/jenkins/install_coreneuron.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/jenkins/install_coreneuron.sh b/tests/jenkins/install_coreneuron.sh index c2f3b614d..87c406d6e 100755 --- a/tests/jenkins/install_coreneuron.sh +++ b/tests/jenkins/install_coreneuron.sh @@ -10,7 +10,7 @@ reportinglib_dir=$(spack location --install-dir --latest reportinglib%intel) CORENRN_TYPE="$1" if [ "${CORENRN_TYPE}" = "GPU-non-unified" ] || [ "${CORENRN_TYPE}" = "GPU-unified" ]; then - module load nvhpc/20.9 cuda/10.1.243 hpe-mpi cmake boost + module load pgi/19.10 cuda hpe-mpi cmake boost mkdir build_${CORENRN_TYPE} else From 76d5d9d37dda629479ccf673cef69906fa4a5bc5 Mon Sep 17 00:00:00 2001 From: Pramod Kumbhar Date: Wed, 6 Jan 2021 08:03:29 +0100 Subject: [PATCH 07/17] back to nvhpc 20.9 as PGI 19.10 doesn't work with current GPU driver --- tests/jenkins/install_coreneuron.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/jenkins/install_coreneuron.sh b/tests/jenkins/install_coreneuron.sh index 87c406d6e..c2f3b614d 100755 --- a/tests/jenkins/install_coreneuron.sh +++ b/tests/jenkins/install_coreneuron.sh @@ -10,7 +10,7 @@ reportinglib_dir=$(spack location --install-dir --latest reportinglib%intel) CORENRN_TYPE="$1" if [ "${CORENRN_TYPE}" = "GPU-non-unified" ] || [ "${CORENRN_TYPE}" = "GPU-unified" ]; then - module load pgi/19.10 cuda hpe-mpi cmake boost + module load nvhpc/20.9 cuda/10.1.243 hpe-mpi cmake boost mkdir build_${CORENRN_TYPE} else From 38323e4d4ec5d6c545d0d4ba7258444f80afd309 Mon Sep 17 00:00:00 2001 From: Pramod S Kumbhar Date: Wed, 6 Jan 2021 21:09:28 +0100 Subject: [PATCH 08/17] use spack branch only if specified --- tests/jenkins/Jenkinsfile | 2 +- tests/jenkins/spack_setup.sh | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/jenkins/Jenkinsfile b/tests/jenkins/Jenkinsfile index bd5d6d1be..9f9d50bbf 100644 --- a/tests/jenkins/Jenkinsfile +++ b/tests/jenkins/Jenkinsfile @@ -5,7 +5,7 @@ pipeline { parameters { string(name: 'sha1', defaultValue: 'master', description: 'What branch of CoreNeuron to test.') - string(name: 'SPACK_BRANCH', defaultValue: 'develop', + string(name: 'SPACK_BRANCH', defaultValue: '', description: 'Which branch of spack to use.') string(name: 'NEURON_BRANCH', defaultValue: '', description: 'Which branch of neuron to use. For master branch (neuron@develop) leave this parameter blank.') diff --git a/tests/jenkins/spack_setup.sh b/tests/jenkins/spack_setup.sh index 0acd733de..9113d8ed1 100755 --- a/tests/jenkins/spack_setup.sh +++ b/tests/jenkins/spack_setup.sh @@ -16,10 +16,16 @@ install_spack() ( mkdir -p $BASEDIR && cd $BASEDIR rm -rf .spack # CLEANUP SPACK CONFIGS SPACK_REPO=https://github.com/BlueBrain/spack.git - SPACK_BRANCH=${SPACK_BRANCH:-"develop"} - echo "Installing SPACK. Cloning $SPACK_REPO $SPACK_ROOT --depth 1 -b $SPACK_BRANCH" - git clone $SPACK_REPO $SPACK_ROOT --depth 1 -b $SPACK_BRANCH + # cloning branch option + BRANCH_OPT="" + if [ -n "$SPACK_BRANCH" ]; then + BRANCH_OPT="-b ${SPACK_BRANCH}" + fi + + echo "Installing SPACK. Cloning $SPACK_REPO $SPACK_ROOT --depth 1 $BRANCH_OPT" + git clone $SPACK_REPO $SPACK_ROOT --depth 1 $BRANCH_OPT + # Use BBP configs mkdir -p $SPACK_ROOT/etc/spack/defaults/linux cp /gpfs/bbp.cscs.ch/apps/hpc/jenkins/config/*.yaml $SPACK_ROOT/etc/spack/ From c6526e20b0b88146654086df20e96ad78a11b76b Mon Sep 17 00:00:00 2001 From: Pramod Kumbhar Date: Thu, 7 Jan 2021 00:19:24 +0100 Subject: [PATCH 09/17] try same version as master --- tests/jenkins/install_coreneuron.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/jenkins/install_coreneuron.sh b/tests/jenkins/install_coreneuron.sh index c2f3b614d..87c406d6e 100755 --- a/tests/jenkins/install_coreneuron.sh +++ b/tests/jenkins/install_coreneuron.sh @@ -10,7 +10,7 @@ reportinglib_dir=$(spack location --install-dir --latest reportinglib%intel) CORENRN_TYPE="$1" if [ "${CORENRN_TYPE}" = "GPU-non-unified" ] || [ "${CORENRN_TYPE}" = "GPU-unified" ]; then - module load nvhpc/20.9 cuda/10.1.243 hpe-mpi cmake boost + module load pgi/19.10 cuda hpe-mpi cmake boost mkdir build_${CORENRN_TYPE} else From 29ed9a497a20d98e969ff95327f89e714eb537dc Mon Sep 17 00:00:00 2001 From: Pramod Kumbhar Date: Thu, 7 Jan 2021 06:45:20 +0100 Subject: [PATCH 10/17] minimize builds --- tests/jenkins/Jenkinsfile | 17 ++++++++++++----- tests/jenkins/install_coreneuron.sh | 4 ++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/jenkins/Jenkinsfile b/tests/jenkins/Jenkinsfile index 9f9d50bbf..d982dcc07 100644 --- a/tests/jenkins/Jenkinsfile +++ b/tests/jenkins/Jenkinsfile @@ -18,6 +18,7 @@ pipeline { stages { + /* stage('install Spack'){ steps { sh 'source $WORKSPACE/tests/jenkins/spack_setup.sh' @@ -29,9 +30,11 @@ pipeline { sh 'sh tests/jenkins/install_neuron_reportinglib.sh' } } + */ stage('build CoreNeuron'){ parallel{ + /* stage('AoS'){ stages{ stage('build'){ @@ -61,6 +64,7 @@ pipeline { } } } + */ stage('GPU-non-unified'){ stages{ @@ -94,6 +98,7 @@ pipeline { } } + /* stage('neuron_direct'){ parallel{ stage('AoS'){ @@ -107,8 +112,9 @@ pipeline { } } } - } + }*/ + /* stage('checkout tests'){ parallel{ stage('testcorenrn'){ @@ -133,8 +139,9 @@ pipeline { } } } - } + }*/ + /* stage('nrnivmodl'){ parallel{ stage('nrnivmodl testcorenrn'){ @@ -183,8 +190,8 @@ pipeline { } } } - } - + }*/ + /* stage('testcorenrn'){ parallel{ stage('deriv'){ @@ -393,7 +400,7 @@ pipeline { } } } - } + }*/ } post { always { diff --git a/tests/jenkins/install_coreneuron.sh b/tests/jenkins/install_coreneuron.sh index 87c406d6e..eadc1b430 100755 --- a/tests/jenkins/install_coreneuron.sh +++ b/tests/jenkins/install_coreneuron.sh @@ -5,7 +5,7 @@ set -x source ${JENKINS_DIR:-.}/_env_setup.sh -reportinglib_dir=$(spack location --install-dir --latest reportinglib%intel) +#reportinglib_dir=$(spack location --install-dir --latest reportinglib%intel) CORENRN_TYPE="$1" @@ -66,7 +66,7 @@ elif [ "${CORENRN_TYPE}" = "AoS" ] || [ "${CORENRN_TYPE}" = "SoA" ]; then -DCMAKE_BUILD_TYPE=Debug \ -DCORENRN_ENABLE_SOA=$CORENRN_ENABLE_SOA \ -DCORENRN_ENABLE_OPENMP=$ENABLE_OPENMP \ - -DCORENRN_ENABLE_BIN_REPORTS=ON \ + -DCORENRN_ENABLE_BIN_REPORTS=OFF \ -DCMAKE_PREFIX_PATH=$reportinglib_dir \ -DTEST_MPI_EXEC_BIN="mpirun" \ -DTEST_EXEC_PREFIX="mpirun;-n;2" \ From fbfa72d88b4afc644ad789e448760f5d7423969a Mon Sep 17 00:00:00 2001 From: Pramod Kumbhar Date: Thu, 7 Jan 2021 06:57:36 +0100 Subject: [PATCH 11/17] use nvhpc/20.9 cuda/10.1.243 --- tests/jenkins/install_coreneuron.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/jenkins/install_coreneuron.sh b/tests/jenkins/install_coreneuron.sh index eadc1b430..a94c1c59b 100755 --- a/tests/jenkins/install_coreneuron.sh +++ b/tests/jenkins/install_coreneuron.sh @@ -10,7 +10,7 @@ source ${JENKINS_DIR:-.}/_env_setup.sh CORENRN_TYPE="$1" if [ "${CORENRN_TYPE}" = "GPU-non-unified" ] || [ "${CORENRN_TYPE}" = "GPU-unified" ]; then - module load pgi/19.10 cuda hpe-mpi cmake boost + module load nvhpc/20.9 cuda/10.1.243 hpe-mpi cmake boost mkdir build_${CORENRN_TYPE} else From 631f04c7e03caae31788bc03c31b88ed27712e7c Mon Sep 17 00:00:00 2001 From: Pramod Kumbhar Date: Thu, 7 Jan 2021 07:09:16 +0100 Subject: [PATCH 12/17] move to archive --- tests/jenkins/Jenkinsfile | 8 ++++++-- tests/jenkins/_env_setup.sh | 2 +- tests/jenkins/install_coreneuron.sh | 3 +-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/jenkins/Jenkinsfile b/tests/jenkins/Jenkinsfile index d982dcc07..22abf6942 100644 --- a/tests/jenkins/Jenkinsfile +++ b/tests/jenkins/Jenkinsfile @@ -1,6 +1,9 @@ pipeline { agent { - label 'bb5 && !bb5-07' + node { + label 'bb5 && !bb5-07' + customWorkspace "/gpfs/bbp.cscs.ch/project/proj16/kumbhar/pramod_scratch/jenkins/${BUILD_TAG}" + } } parameters { string(name: 'sha1', defaultValue: 'master', @@ -402,9 +405,10 @@ pipeline { } }*/ } + /* post { always { cleanWs() } - } + }*/ } diff --git a/tests/jenkins/_env_setup.sh b/tests/jenkins/_env_setup.sh index e5007b71e..12bd352a6 100644 --- a/tests/jenkins/_env_setup.sh +++ b/tests/jenkins/_env_setup.sh @@ -1,7 +1,7 @@ # Upstream modules unset MODULEPATH source /gpfs/bbp.cscs.ch/apps/hpc/jenkins/config/modules.sh -module load unstable +module load archive/2020-12 # Local spack BUILD_HOME="${WORKSPACE}/BUILD_HOME" diff --git a/tests/jenkins/install_coreneuron.sh b/tests/jenkins/install_coreneuron.sh index a94c1c59b..1eb4c8414 100755 --- a/tests/jenkins/install_coreneuron.sh +++ b/tests/jenkins/install_coreneuron.sh @@ -10,8 +10,7 @@ source ${JENKINS_DIR:-.}/_env_setup.sh CORENRN_TYPE="$1" if [ "${CORENRN_TYPE}" = "GPU-non-unified" ] || [ "${CORENRN_TYPE}" = "GPU-unified" ]; then - module load nvhpc/20.9 cuda/10.1.243 hpe-mpi cmake boost - + module load pgi/19.10 cuda hpe-mpi cmake boost mkdir build_${CORENRN_TYPE} else module load boost intel hpe-mpi cmake From 053d73fc2f4dd5b70fbf4df87b47871f009d7cb0 Mon Sep 17 00:00:00 2001 From: Pramod Kumbhar Date: Thu, 7 Jan 2021 22:56:17 +0100 Subject: [PATCH 13/17] Remove use of PG_ACC_BUGS * PG_ACC_BUGS was added when we saw bugs with PGI compiler version 16.3 * Except the case of union, no more real buggy scenario exists * So remove all usages of PG_ACC_BUGS * Remove some hardcoded ion array size by const variable --- CMake/OpenAccHelper.cmake | 3 +-- coreneuron/gpu/nrn_acc_manager.cpp | 3 +-- coreneuron/mechanism/eion.cpp | 19 +++++++------------ coreneuron/mechanism/mechanism.hpp | 4 +++- coreneuron/mechanism/membfunc.hpp | 2 ++ coreneuron/mechanism/register_mech.cpp | 9 +++------ external/mod2c | 2 +- 7 files changed, 18 insertions(+), 24 deletions(-) diff --git a/CMake/OpenAccHelper.cmake b/CMake/OpenAccHelper.cmake index eab63e710..e1858dfbb 100644 --- a/CMake/OpenAccHelper.cmake +++ b/CMake/OpenAccHelper.cmake @@ -26,7 +26,6 @@ if(CORENRN_ENABLE_GPU) if(${CMAKE_C_COMPILER_ID} STREQUAL "PGI") # workaround for old PGI version - set(PGI_ACC_BUG_DEFS -DPG_ACC_BUGS) set(PGI_ACC_FLAGS "-acc") # disable very verbose diagnosis messages and obvious warnings for mod2c set(PGI_DIAG_FLAGS "--diag_suppress 161,177,550") @@ -51,7 +50,7 @@ if(CORENRN_ENABLE_GPU) set(CUDA_PROPAGATE_HOST_FLAGS OFF) set(CUDA_PROFILING_DEF -DCUDA_PROFILING) - set(CORENRN_ACC_GPU_DEFS "${UNIFIED_MEMORY_DEF} ${CUDA_PROFILING_DEF} ${PGI_ACC_BUG_DEFS}") + set(CORENRN_ACC_GPU_DEFS "${UNIFIED_MEMORY_DEF} ${CUDA_PROFILING_DEF}") set(CORENRN_ACC_GPU_FLAGS "${PGI_ACC_FLAGS} ${PGI_DIAG_FLAGS} ${PGI_INLINE_FLAGS}") add_definitions(${CORENRN_ACC_GPU_DEFS}) diff --git a/coreneuron/gpu/nrn_acc_manager.cpp b/coreneuron/gpu/nrn_acc_manager.cpp index 41fb2635a..55a26b523 100644 --- a/coreneuron/gpu/nrn_acc_manager.cpp +++ b/coreneuron/gpu/nrn_acc_manager.cpp @@ -969,8 +969,7 @@ void nrn_ion_global_map_copyto_device() { (double**)acc_copyin(nrn_ion_global_map, sizeof(double*) * nrn_ion_global_map_size); for (int j = 0; j < nrn_ion_global_map_size; j++) { if (nrn_ion_global_map[j]) { - /* @todo: fix this constant size 3 :( */ - double* d_mechmap = (double*)acc_copyin(nrn_ion_global_map[j], 3 * sizeof(double)); + double* d_mechmap = (double*)acc_copyin(nrn_ion_global_map[j], ion_global_map_member_size * sizeof(double)); acc_memcpy_to_device(&(d_data[j]), &d_mechmap, sizeof(double*)); } } diff --git a/coreneuron/mechanism/eion.cpp b/coreneuron/mechanism/eion.cpp index b2cc46dec..e2eee7b53 100644 --- a/coreneuron/mechanism/eion.cpp +++ b/coreneuron/mechanism/eion.cpp @@ -48,21 +48,12 @@ THE POSSIBILITY OF SUCH DAMAGE. #endif #if defined(_OPENACC) -#if defined(PG_ACC_BUGS) #define _PRAGMA_FOR_INIT_ACC_LOOP_ \ _Pragma( \ - "acc parallel loop present(pd[0:_cntml_padded*5], ppd[0:1], nrn_ion_global_map[0:nrn_ion_global_map_size][0:3]) if(nt->compute_gpu)") + "acc parallel loop present(pd[0:_cntml_padded*5], ppd[0:1], nrn_ion_global_map[0:nrn_ion_global_map_size][0:ion_global_map_member_size]) if(nt->compute_gpu)") #define _PRAGMA_FOR_CUR_ACC_LOOP_ \ _Pragma( \ - "acc parallel loop present(pd[0:_cntml_padded*5], nrn_ion_global_map[0:nrn_ion_global_map_size][0:3]) if(nt->compute_gpu) async(stream_id)") -#else -#define _PRAGMA_FOR_INIT_ACC_LOOP_ \ - _Pragma( \ - "acc parallel loop present(pd[0:_cntml_padded*5], ppd[0:1], nrn_ion_global_map[0:nrn_ion_global_map_size]) if(nt->compute_gpu)") -#define _PRAGMA_FOR_CUR_ACC_LOOP_ \ - _Pragma( \ - "acc parallel loop present(pd[0:_cntml_padded*5], nrn_ion_global_map[0:nrn_ion_global_map_size]) if(nt->compute_gpu) async(stream_id)") -#endif + "acc parallel loop present(pd[0:_cntml_padded*5], nrn_ion_global_map[0:nrn_ion_global_map_size][0:ion_global_map_member_size]) if(nt->compute_gpu) async(stream_id)") #define _PRAGMA_FOR_SEC_ORDER_CUR_ACC_LOOP_ \ _Pragma( \ "acc parallel loop present(pd[0:_cntml_padded*5], ni[0:_cntml_actual], _vec_rhs[0:_nt->end]) if(_nt->compute_gpu) async(stream_id)") @@ -74,6 +65,10 @@ THE POSSIBILITY OF SUCH DAMAGE. namespace coreneuron { +// for each ion it refers to internal concentration, external concentration, and charge, +const int ion_global_map_member_size = 3; + + #define nparm 5 static const char* mechanism[] = {/*just a template*/ "0", "na_ion", "ena", "nao", "nai", 0, "ina", "dina_dv_", 0, 0}; @@ -130,7 +125,7 @@ void ion_reg(const char* name, double valence) { } nrn_ion_global_map_size = mechtype + 1; } - nrn_ion_global_map[mechtype] = (double*)emalloc(3 * sizeof(double)); + nrn_ion_global_map[mechtype] = (double*)emalloc(ion_global_map_member_size * sizeof(double)); register_mech((const char**)mechanism, nrn_alloc_ion, nrn_cur_ion, (mod_f_t)0, (mod_f_t)0, (mod_f_t)nrn_init_ion, -1, 1); diff --git a/coreneuron/mechanism/mechanism.hpp b/coreneuron/mechanism/mechanism.hpp index 6ba6e431c..1c1c63d06 100644 --- a/coreneuron/mechanism/mechanism.hpp +++ b/coreneuron/mechanism/mechanism.hpp @@ -33,7 +33,9 @@ THE POSSIBILITY OF SUCH DAMAGE. #include "coreneuron/utils/memory.h" namespace coreneuron { -#if PG_ACC_BUGS +// OpenACC with PGI compiler has issue when union is used and hence use struct +// \todo check if newer PGI versions has resolved this issue +#if defined(_OPENACC) struct ThreadDatum { int i; double* pval; diff --git a/coreneuron/mechanism/membfunc.hpp b/coreneuron/mechanism/membfunc.hpp index 3554c1f1e..7cfd73d69 100644 --- a/coreneuron/mechanism/membfunc.hpp +++ b/coreneuron/mechanism/membfunc.hpp @@ -86,6 +86,7 @@ struct BAMech { extern int nrn_ion_global_map_size; extern double** nrn_ion_global_map; +extern const int ion_global_map_member_size; #define NRNPOINTER \ 4 /* added on to list of mechanism variables.These are \ @@ -94,6 +95,7 @@ pointers which connect variables from other mechanisms via the _ppval array. \ #define _AMBIGUOUS 5 + extern int nrn_get_mechtype(const char*); extern const char* nrn_get_mechname(int); // slow. use memb_func[i].sym if posible extern int register_mech(const char** m, diff --git a/coreneuron/mechanism/register_mech.cpp b/coreneuron/mechanism/register_mech.cpp index 887d372e5..d02eb9154 100644 --- a/coreneuron/mechanism/register_mech.cpp +++ b/coreneuron/mechanism/register_mech.cpp @@ -40,12 +40,9 @@ THE POSSIBILITY OF SUCH DAMAGE. namespace coreneuron { int secondorder = 0; double t, dt, celsius; -#if defined(PG_ACC_BUGS) -// clang-format off - #pragma acc declare copyin(secondorder) - #pragma acc declare copyin(celsius) -// clang-format on -#endif +// declare copyin required for correct initialization +#pragma acc declare copyin(secondorder) +#pragma acc declare copyin(celsius) int rev_dt; using Pfrv = void (*)(); diff --git a/external/mod2c b/external/mod2c index 459a4e374..2fdc2a0a9 160000 --- a/external/mod2c +++ b/external/mod2c @@ -1 +1 @@ -Subproject commit 459a4e37418b42433194e1d25d55943470f61eaf +Subproject commit 2fdc2a0a949195aa06f968256c66e26e3f74342b From 059c1fb15c455bde173deebc9c7af4c05011ecb5 Mon Sep 17 00:00:00 2001 From: Pramod Kumbhar Date: Thu, 7 Jan 2021 23:34:44 +0100 Subject: [PATCH 14/17] revert temporary changes --- tests/jenkins/Jenkinsfile | 22 +++++++--------------- tests/jenkins/_env_setup.sh | 2 +- tests/jenkins/install_coreneuron.sh | 6 +++--- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/tests/jenkins/Jenkinsfile b/tests/jenkins/Jenkinsfile index 22abf6942..6ce5d3f54 100644 --- a/tests/jenkins/Jenkinsfile +++ b/tests/jenkins/Jenkinsfile @@ -2,7 +2,6 @@ pipeline { agent { node { label 'bb5 && !bb5-07' - customWorkspace "/gpfs/bbp.cscs.ch/project/proj16/kumbhar/pramod_scratch/jenkins/${BUILD_TAG}" } } parameters { @@ -21,7 +20,6 @@ pipeline { stages { - /* stage('install Spack'){ steps { sh 'source $WORKSPACE/tests/jenkins/spack_setup.sh' @@ -33,11 +31,9 @@ pipeline { sh 'sh tests/jenkins/install_neuron_reportinglib.sh' } } - */ stage('build CoreNeuron'){ parallel{ - /* stage('AoS'){ stages{ stage('build'){ @@ -67,7 +63,6 @@ pipeline { } } } - */ stage('GPU-non-unified'){ stages{ @@ -101,7 +96,6 @@ pipeline { } } - /* stage('neuron_direct'){ parallel{ stage('AoS'){ @@ -115,9 +109,8 @@ pipeline { } } } - }*/ + } - /* stage('checkout tests'){ parallel{ stage('testcorenrn'){ @@ -142,9 +135,8 @@ pipeline { } } } - }*/ + } - /* stage('nrnivmodl'){ parallel{ stage('nrnivmodl testcorenrn'){ @@ -193,8 +185,8 @@ pipeline { } } } - }*/ - /* + } + stage('testcorenrn'){ parallel{ stage('deriv'){ @@ -403,12 +395,12 @@ pipeline { } } } - }*/ + } } - /* + post { always { cleanWs() } - }*/ + } } diff --git a/tests/jenkins/_env_setup.sh b/tests/jenkins/_env_setup.sh index 12bd352a6..e5007b71e 100644 --- a/tests/jenkins/_env_setup.sh +++ b/tests/jenkins/_env_setup.sh @@ -1,7 +1,7 @@ # Upstream modules unset MODULEPATH source /gpfs/bbp.cscs.ch/apps/hpc/jenkins/config/modules.sh -module load archive/2020-12 +module load unstable # Local spack BUILD_HOME="${WORKSPACE}/BUILD_HOME" diff --git a/tests/jenkins/install_coreneuron.sh b/tests/jenkins/install_coreneuron.sh index 1eb4c8414..87249cd3e 100755 --- a/tests/jenkins/install_coreneuron.sh +++ b/tests/jenkins/install_coreneuron.sh @@ -5,12 +5,12 @@ set -x source ${JENKINS_DIR:-.}/_env_setup.sh -#reportinglib_dir=$(spack location --install-dir --latest reportinglib%intel) +reportinglib_dir=$(spack location --install-dir --latest reportinglib%intel) CORENRN_TYPE="$1" if [ "${CORENRN_TYPE}" = "GPU-non-unified" ] || [ "${CORENRN_TYPE}" = "GPU-unified" ]; then - module load pgi/19.10 cuda hpe-mpi cmake boost + module load nvhpc/20.9 cuda/11.1.0 hpe-mpi cmake boost mkdir build_${CORENRN_TYPE} else module load boost intel hpe-mpi cmake @@ -65,7 +65,7 @@ elif [ "${CORENRN_TYPE}" = "AoS" ] || [ "${CORENRN_TYPE}" = "SoA" ]; then -DCMAKE_BUILD_TYPE=Debug \ -DCORENRN_ENABLE_SOA=$CORENRN_ENABLE_SOA \ -DCORENRN_ENABLE_OPENMP=$ENABLE_OPENMP \ - -DCORENRN_ENABLE_BIN_REPORTS=OFF \ + -DCORENRN_ENABLE_BIN_REPORTS=ON \ -DCMAKE_PREFIX_PATH=$reportinglib_dir \ -DTEST_MPI_EXEC_BIN="mpirun" \ -DTEST_EXEC_PREFIX="mpirun;-n;2" \ From 68b42b980c9f348735fd22787d820e7dcdfd4c74 Mon Sep 17 00:00:00 2001 From: Pramod Kumbhar Date: Thu, 7 Jan 2021 23:37:43 +0100 Subject: [PATCH 15/17] update mod2c to master --- external/mod2c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/mod2c b/external/mod2c index 2fdc2a0a9..a95bd2b42 160000 --- a/external/mod2c +++ b/external/mod2c @@ -1 +1 @@ -Subproject commit 2fdc2a0a949195aa06f968256c66e26e3f74342b +Subproject commit a95bd2b429807fe2ebe58b823e966c9719679a7a From 57f13d848723041bf2eca4b4dcfd7a9a4081fef8 Mon Sep 17 00:00:00 2001 From: Pramod Kumbhar Date: Fri, 8 Jan 2021 00:12:23 +0100 Subject: [PATCH 16/17] use archive modules again --- tests/jenkins/install_coreneuron.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/jenkins/install_coreneuron.sh b/tests/jenkins/install_coreneuron.sh index 87249cd3e..896718b46 100755 --- a/tests/jenkins/install_coreneuron.sh +++ b/tests/jenkins/install_coreneuron.sh @@ -10,7 +10,9 @@ reportinglib_dir=$(spack location --install-dir --latest reportinglib%intel) CORENRN_TYPE="$1" if [ "${CORENRN_TYPE}" = "GPU-non-unified" ] || [ "${CORENRN_TYPE}" = "GPU-unified" ]; then - module load nvhpc/20.9 cuda/11.1.0 hpe-mpi cmake boost + # PGI compiler issue in unstable : BSD-204 + module unload unstable && module load archive/2020-12 + module load pgi/19.10 cuda hpe-mpi cmake boost mkdir build_${CORENRN_TYPE} else module load boost intel hpe-mpi cmake From d6e909c9d50be5fe10330bd54e0b92eb685f76e9 Mon Sep 17 00:00:00 2001 From: Pramod Kumbhar Date: Fri, 8 Jan 2021 07:55:16 +0100 Subject: [PATCH 17/17] mods files and cpp translated files are in same x86_64/corenrn/mod2c directory --- extra/nrnivmodl-core.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extra/nrnivmodl-core.in b/extra/nrnivmodl-core.in index 697573517..6c0cd0568 100755 --- a/extra/nrnivmodl-core.in +++ b/extra/nrnivmodl-core.in @@ -100,7 +100,7 @@ else fi # temporary directory where mod files will be copied -temp_mod_dir=@CMAKE_HOST_SYSTEM_PROCESSOR@/core/mods +temp_mod_dir=@CMAKE_HOST_SYSTEM_PROCESSOR@/corenrn/mod2c mkdir -p $temp_mod_dir # copy mod files with include files. note that ${ROOT_DIR}/share @@ -109,7 +109,7 @@ set +e for mod_dir in ${ROOT_DIR}/share/modfile $params_MODS_PATH; do # copy mod files and include files - files=`ls $mod_dir/*.mod $mod_dir/*.inc 2>/dev/null` + files=`ls $mod_dir/*.mod $mod_dir/*.inc $mod_dir/*.h* 2>/dev/null` for f in $files; do # copy mod files only if it's changed (to avoid rebuild)