MWE of skbuild issues (port)

projects/hello-complex/CMakeLists.txt

@@ -0,0 +1,48 @@
+cmake_minimum_required(VERSION 3.13.0)
+project(my_skb_mod LANGUAGES C)
+
+
+find_package(PythonInterp REQUIRED)
+find_package(PythonLibs REQUIRED)
+
+
+###
+# Private helper function to execute `python -c "<cmd>"`
+#
+# Runs a python command and populates an outvar with the result of stdout.
+# Be careful of indentation if `cmd` is multiline.
+#
+function(pycmd outvar cmd)
+  execute_process(
+    COMMAND "${PYTHON_EXECUTABLE}" -c "${cmd}"
+    RESULT_VARIABLE _exitcode
+    OUTPUT_VARIABLE _output)
+  if(NOT ${_exitcode} EQUAL 0)
+    message(ERROR "Failed when running python code: \"\"\"
+${cmd}\"\"\"")
+    message(FATAL_ERROR "Python command failed with error code: ${_exitcode}")
+  endif()
+  # Remove supurflous newlines (artifacts of print)
+  string(STRIP "${_output}" _output)
+  set(${outvar} "${_output}" PARENT_SCOPE)
+endfunction()
+
+###
+# Find scikit-build and include its cmake resource scripts
+#
+if (NOT SKBUILD)
+  pycmd(skbuild_location "import os, skbuild; print(os.path.dirname(skbuild.__file__))")
+  set(skbuild_cmake_dir "${skbuild_location}/resources/cmake")
+  # If skbuild is not the driver, then we need to include its utilities in our CMAKE_MODULE_PATH
+  list(APPEND CMAKE_MODULE_PATH ${skbuild_cmake_dir})
+endif()
+
+find_package(PythonExtensions REQUIRED)
+find_package(Cython REQUIRED)
+find_package(NumPy REQUIRED)
+
+# Backend C library
+add_subdirectory("src/cxx")
+
+# Cython library
+add_subdirectory("src/python/my_skb_mod")

projects/hello-complex/demo_issue.sh

@@ -0,0 +1,79 @@
+
+run_common_module_tests(){
+    echo "Running common tests of the installed module"
+
+    echo "Package Version: $(python -c 'import my_skb_mod; print(my_skb_mod.__version__)')"
+    echo "Python Package Location: $(python -c 'import my_skb_mod; print(my_skb_mod.__file__)')"
+    echo "Compiled Cython Module: $(python -c 'from my_skb_mod import _myalgo_cython; print(_myalgo_cython.__file__)')"
+    # Test that we can access the module resource
+    python -c "import my_skb_mod; print(my_skb_mod.submod.get_module_resouce())"
+
+    echo "Finished common tests of the installed module"
+
+    python -c "import my_skb_mod; print(my_skb_mod.__doc__)"
+
+
+}
+
+
+#######################
+# WHEEL MODE TEST
+#######################
+
+# First test with a regular wheel
+pip wheel .
+WHEEL_FPATH="$(ls my_skb_mod-*.whl)"
+echo "WHEEL_FPATH = $WHEEL_FPATH"
+# Should see the submodule resource here, but we don't
+unzip -l "$WHEEL_FPATH"
+
+# Install the wheel
+pip uninstall -y "my_skb_mod" || echo "already uninstalled"
+pip install "$WHEEL_FPATH"
+
+# Run the module tests
+# This does not include the package data correctly
+run_common_module_tests
+
+
+#######################
+# DEVELOPMENT MODE TEST
+#######################
+
+# Clean up
+pip uninstall -y "my_skb_mod" || echo "already uninstalled"
+rm -rf _skbuild || echo "already clean"
+rm -rf src/python/my_skb_mod/*.so || echo "already clean"
+
+# Test in development mode
+# FIXME: running ``pip install -e .`` multiple times breaks!
+pip install --verbose -e .
+
+# The egg link should point to
+# .../issue-xxx-multi-package-with-package-dir/src/python
+#
+# NOT
+#
+# .../issue-xxx-multi-package-with-package-dir/
+SITE_PACKAGE_DPATH="$(python -c 'import distutils.sysconfig; print(distutils.sysconfig.get_python_lib())')"
+echo "SITE_PACKAGE_DPATH = $SITE_PACKAGE_DPATH"
+cat "$SITE_PACKAGE_DPATH/my_skb_mod.egg-link"
+# but even with the packages=[''] hack, there seems to be an extra ../../.. term
+# not sure what's going on.
+
+run_common_module_tests
+
+
+
+#######################
+# DEMO MULTIPLE EDIBALE PIP INSTALL ISSUE
+#######################
+
+# Maybe this isn't an issue here? It is an issue in the project I'm basing this
+# on cant run this command multiple times. I dont know why.
+# This only seeps to break if the "packages" directory is specified as
+# `packages=['my_skb_mod']`. Otherwise it does seem to work
+pip uninstall -y "my_skb_mod" || echo "already uninstalled"
+pip uninstall -y "my_skb_mod" || echo "already uninstalled"
+pip install -e .
+pip install -e .

projects/hello-complex/setup.py

@@ -0,0 +1,24 @@
+"""
+MWE for issue
+"""
+from skbuild import setup
+
+if __name__ == '__main__':
+    setup(
+        package_dir={
+            '': 'src/python/',
+        },
+        name='my_skb_mod',
+        version='0.1.0',
+        # develop mode doesnt work without this hack
+        # packages=['', 'my_skb_mod', 'my_skb_mod.submod'],  # sort of works? but also warns?
+        packages=['my_skb_mod', 'my_skb_mod.submod'],  # works for wheel install
+        # packages=['my_skb_mod'],  # works for develop install
+
+        # This package data seems like it is not respected.
+        package_data={
+            'my_skb_mod.submod': [
+                'module_resouce.json'
+            ],
+        },
+    )

projects/hello-complex/src/cxx/CMakeLists.txt

@@ -0,0 +1,67 @@
+# TODO: Can we make this work in either shared or static mode?
+
+option(BUILD_MECL "Enable MECL (my extern c lib) library" TRUE)
+if (BUILD_MECL)
+
+  set(THREADS_PREFER_PTHREAD_FLAG ON)
+
+  # TODO: include external deps in mecl
+
+  # Find required external libraries
+  find_package(Threads REQUIRED)
+  #find_package(ZLIB REQUIRED)
+  #find_package(GSL REQUIRED)
+  #find_package(OpenMP)
+
+  # Corresponds to FFLAGS
+  #set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -g -fPIC")
+
+  # Hacks:
+  #set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC")
+
+  #add_library(GLMnet STATIC "GLMnet.f")
+
+  set(MECL_MODULE_NAME "mecl")
+  # Add other C sources
+  list(APPEND mecl_sources "my_extern_clib.c")
+  ## Create C++ library. Specify include dirs and link libs as normal
+  #add_library(${MECL_MODULE_NAME} SHARED ${mecl_sources})
+
+  # HACK: statically link the library to the cython module because
+  # I'm having trouble making the shared library work.
+  add_library(${MECL_MODULE_NAME} STATIC ${mecl_sources})
+
+  #target_include_directories(
+  #  ${MECL_MODULE_NAME}
+  #  PUBLIC
+  #      ${NumPy_INCLUDE_DIRS}
+  #      ${PYTHON_INCLUDE_DIRS}
+  #      ${CMAKE_CURRENT_SOURCE_DIR}
+  #)
+
+  #message(STATUS "OpenMP::OpenMP_C = ${OpenMP::OpenMP_C}")
+  #message(STATUS "ZLIB::ZLIB = ${ZLIB::ZLIB}")
+  #message(STATUS "Threads::Threads = ${Threads::Threads}")
+  message(STATUS "Threads = ${Threads}")
+
+  target_link_libraries(
+    ${MECL_MODULE_NAME} PUBLIC
+    #ZLIB::ZLIB
+    Threads::Threads
+  )
+
+  #target_compile_definitions(${SCCD_MODULE_NAME} PUBLIC
+  #  "NPY_NO_DEPRECATED_API"
+  #  #"NPY_1_7_API_VERSION=0x00000007"
+  #  )
+
+  # Transform the C++ library into an importable python module
+  #python_extension_module(${SCCD_MODULE_NAME})
+
+  # Install the C++ module to the correct relative location
+  # (this will be an inplace build if you use `pip install -e`)
+  #file(RELATIVE_PATH _install_dest "${CMAKE_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}")
+  # Is this right?
+  #install(TARGETS ${SCCD_MODULE_NAME} LIBRARY DESTINATION "${_install_dest}../tool/python/pycold")
+
+endif()

projects/hello-complex/src/cxx/my_extern_clib.c

@@ -0,0 +1,9 @@
+#include "my_extern_clib.h"
+
+long external_c_function(long* data_view, int size){
+    long result = 0;
+    for (int i = 0; i < size; i++){
+        result += data_view[i];
+    }
+    return result;
+}

projects/hello-complex/src/cxx/my_extern_clib.h

@@ -0,0 +1,6 @@
+#ifndef MYEXTERN_CLIB_H
+#define MYEXTERN_CLIB_H
+
+long external_c_function(long* data_view, int size);
+
+#endif

projects/hello-complex/src/python/my_skb_mod/CMakeLists.txt

@@ -0,0 +1,47 @@
+# Setup basic python stuff and ensure we have skbuild
+
+option(BUILD_MYALGO "Enable _myalgo_cython module" TRUE)
+if (BUILD_MYALGO)
+
+  set(cython_source "_myalgo_cython.pyx")
+  set(MYALGO_MODULE_NAME "_myalgo_cython")
+
+  # Translate Cython into C/C++
+  add_cython_target(${MYALGO_MODULE_NAME} "${cython_source}" C OUTPUT_VAR sources)
+
+  # Add other C sources
+  list(APPEND sources )
+
+  # Create C++ library. Specify include dirs and link libs as normal
+  add_library(${MYALGO_MODULE_NAME} MODULE ${sources})
+  target_include_directories(
+    ${MYALGO_MODULE_NAME}
+    PUBLIC
+        ${NumPy_INCLUDE_DIRS}
+        ${PYTHON_INCLUDE_DIR}
+        ${CMAKE_CURRENT_SOURCE_DIR}
+  )
+
+  # TODO: not sure why this isn't set in the global scope?
+  # Hack around it: just hard code the module name
+  set(MECL_MODULE_NAME "mecl")
+
+  # TODO: linking to the SCCD shared object isn't working 100% yet.
+  target_link_libraries(${MYALGO_MODULE_NAME} ${MECL_MODULE_NAME})
+
+  target_compile_definitions(${MYALGO_MODULE_NAME} PUBLIC
+    "NPY_NO_DEPRECATED_API"
+    #"NPY_1_7_API_VERSION=0x00000007"
+    )
+
+  # Transform the C++ library into an importable python module
+  python_extension_module(${MYALGO_MODULE_NAME})
+
+  # My "normal" method of setting install targets does not seem to work here. Hacking it.
+  # NOTE: skbuild *seems* to place libraries in a data dir *unless* the install destination
+  # corresponds exactly to the <package_dir>/<package_name> specified implicitly in setup.py
+  set(my_skb_mod_install_dest "src/python/my_skb_mod")
+
+  install(TARGETS ${MYALGO_MODULE_NAME} LIBRARY DESTINATION "${my_skb_mod_install_dest}")
+
+endif()

projects/hello-complex/src/python/my_skb_mod/__init__.py

@@ -0,0 +1,18 @@
+"""
+My ScikitBuild Module Example
+"""
+__version__ = '0.1.0'
+
+try:
+    from . import submod
+except Exception:
+    print('ERROR submod NOT INCLUDED IN PACAKGE!')
+    submod = None
+
+try:
+    from ._my_skb_mod_cython import cython_func
+except Exception:
+    print('ERROR _my_skb_mod_cython NOT INCLUDED IN PACAKGE!')
+    cython_func = None
+
+__all__ = ['cython_func', 'submod']

projects/hello-complex/src/python/my_skb_mod/_myalgo_cython.pyx

@@ -0,0 +1,17 @@
+import numpy as np
+cimport numpy as np
+
+
+cdef extern from "../../cxx/my_extern_clib.h":
+    cdef long external_c_function(long* data_view, int size);
+
+
+def cython_func(np.ndarray[np.int64_t, ndim=1] data):
+    """
+    This is a docstring
+    """
+    cdef long [:] data_view = data
+    cdef int size = len(data)
+    cdef long result = 0;
+    result = external_c_function(&data_view[0], size)
+    return result

projects/hello-complex/src/python/my_skb_mod/submod/__init__.py

@@ -0,0 +1,6 @@
+def get_module_resource():
+    from importlib import resources as importlib_resources
+    import json
+    file = importlib_resources.open_text('my_skb_mod.submod', 'module_resource.json')
+    data = json.load(file)
+    return data

projects/hello-complex/src/python/my_skb_mod/submod/module_resource.json

@@ -0,0 +1,3 @@
+{
+    "some-key": "some-value"
+}