recipes: add scipy support (#2370)

* add scipy support
* flake8
* remove ccache check
* remove lapack, scipy from ci due to missing gfortran in CI machines
* better recipe names, imports

Author: mzakharo

ci/constants.py

@@ -26,6 +26,8 @@ class TargetPython(Enum):
     # mpmath package with a version >= 0.19 required
     'sympy',
     'vlc',
+    # need extra gfortran NDK system add-on
+    'lapack', 'scipy',
 ])
 
 BROKEN_RECIPES = {

pythonforandroid/recipes/lapack/__init__.py

@@ -0,0 +1,53 @@
+from pythonforandroid.recipe import Recipe
+from pythonforandroid.logger import shprint
+from pythonforandroid.util import current_directory, ensure_dir, BuildInterruptingException
+from multiprocessing import cpu_count
+from os.path import join
+import sh
+
+
+class LapackRecipe(Recipe):
+
+    name = 'lapack'
+    version = 'v3.9.0'
+    url = 'https://github.com/Reference-LAPACK/lapack/archive/{version}.tar.gz'
+    libdir = 'build/install/lib'
+    built_libraries = {'libblas.so': libdir, 'liblapack.so': libdir, 'libcblas.so': libdir}
+    need_stl_shared = True
+
+    def get_recipe_env(self, arch):
+        env = super().get_recipe_env(arch)
+        sysroot = f"{self.ctx.ndk_dir}/platforms/{env['NDK_API']}/{arch.platform_dir}"
+        FC = f"{env['TOOLCHAIN_PREFIX']}-gfortran"
+        env['FC'] = f'{FC} --sysroot={sysroot}'
+        if sh.which(FC) is None:
+            raise BuildInterruptingException(f"{FC} not found. See https://github.com/mzakharo/android-gfortran")
+        return env
+
+    def build_arch(self, arch):
+        source_dir = self.get_build_dir(arch.arch)
+        build_target = join(source_dir, 'build')
+        install_target = join(build_target, 'install')
+
+        ensure_dir(build_target)
+        with current_directory(build_target):
+            env = self.get_recipe_env(arch)
+            shprint(sh.rm, '-rf', 'CMakeFiles/', 'CMakeCache.txt', _env=env)
+            shprint(sh.cmake, source_dir,
+                    '-DCMAKE_SYSTEM_NAME=Android',
+                    '-DCMAKE_POSITION_INDEPENDENT_CODE=1',
+                    '-DCMAKE_ANDROID_ARCH_ABI={arch}'.format(arch=arch.arch),
+                    '-DCMAKE_ANDROID_NDK=' + self.ctx.ndk_dir,
+                    '-DCMAKE_BUILD_TYPE=Release',
+                    '-DCMAKE_INSTALL_PREFIX={}'.format(install_target),
+                    '-DANDROID_ABI={arch}'.format(arch=arch.arch),
+                    '-DANDROID_ARM_NEON=ON',
+                    '-DENABLE_NEON=ON',
+                    '-DCBLAS=ON',
+                    '-DBUILD_SHARED_LIBS=ON',
+                    _env=env)
+            shprint(sh.make, '-j' + str(cpu_count()), _env=env)
+            shprint(sh.make, 'install', _env=env)
+
+
+recipe = LapackRecipe()

pythonforandroid/recipes/numpy/__init__.py

@@ -1,6 +1,10 @@
 from pythonforandroid.recipe import CompiledComponentsPythonRecipe
+from pythonforandroid.logger import shprint, info
+from pythonforandroid.util import current_directory
 from multiprocessing import cpu_count
 from os.path import join
+import glob
+import sh
 
 
 class NumpyRecipe(CompiledComponentsPythonRecipe):
@@ -9,23 +13,47 @@ class NumpyRecipe(CompiledComponentsPythonRecipe):
     url = 'https://pypi.python.org/packages/source/n/numpy/numpy-{version}.zip'
     site_packages_name = 'numpy'
     depends = ['setuptools', 'cython']
+    install_in_hostpython = True
+    call_hostpython_via_targetpython = False
 
     patches = [
+        join('patches', 'hostnumpy-xlocale.patch'),
+        join('patches', 'remove-default-paths.patch'),
         join('patches', 'add_libm_explicitly_to_build.patch'),
-        join('patches', 'do_not_use_system_libs.patch'),
-        join('patches', 'remove_unittest_call.patch'),
+        join('patches', 'compiler_cxx_fix.patch'),
         ]
 
-    call_hostpython_via_targetpython = False
+    def _build_compiled_components(self, arch):
+        info('Building compiled components in {}'.format(self.name))
+
+        env = self.get_recipe_env(arch)
+        with current_directory(self.get_build_dir(arch.arch)):
+            hostpython = sh.Command(self.real_hostpython_location)
+            if self.install_in_hostpython:
+                shprint(hostpython, 'setup.py', 'clean', '--all', '--force', _env=env)
+            hostpython = sh.Command(self.hostpython_location)
+            shprint(hostpython, 'setup.py', self.build_cmd, '-v',
+                    _env=env, *self.setup_extra_args)
+            build_dir = glob.glob('build/lib.*')[0]
+            shprint(sh.find, build_dir, '-name', '"*.o"', '-exec',
+                    env['STRIP'], '{}', ';', _env=env)
+
+    def _rebuild_compiled_components(self, arch, env):
+        info('Rebuilding compiled components in {}'.format(self.name))
+
+        hostpython = sh.Command(self.real_hostpython_location)
+        shprint(hostpython, 'setup.py', 'clean', '--all', '--force', _env=env)
+        shprint(hostpython, 'setup.py', self.build_cmd, '-v', _env=env,
+                *self.setup_extra_args)
 
     def build_compiled_components(self, arch):
         self.setup_extra_args = ['-j', str(cpu_count())]
-        super().build_compiled_components(arch)
+        self._build_compiled_components(arch)
         self.setup_extra_args = []
 
     def rebuild_compiled_components(self, arch, env):
         self.setup_extra_args = ['-j', str(cpu_count())]
-        super().rebuild_compiled_components(arch, env)
+        self._rebuild_compiled_components(arch, env)
         self.setup_extra_args = []
 
 

pythonforandroid/recipes/numpy/patches/compiler_cxx_fix.patch

@@ -0,0 +1,20 @@
+diff --git a/numpy/distutils/ccompiler.py b/numpy/distutils/ccompiler.py
+index 6438790..a5f1527 100644
+--- a/numpy/distutils/ccompiler.py
++++ b/numpy/distutils/ccompiler.py
+@@ -686,13 +686,13 @@ def CCompiler_cxx_compiler(self):
+         return self
+ 
+     cxx = copy(self)
+-    cxx.compiler_so = [cxx.compiler_cxx[0]] + cxx.compiler_so[1:]
++    cxx.compiler_so = cxx.compiler_cxx + cxx.compiler_so[1:]
+     if sys.platform.startswith('aix') and 'ld_so_aix' in cxx.linker_so[0]:
+         # AIX needs the ld_so_aix script included with Python
+         cxx.linker_so = [cxx.linker_so[0], cxx.compiler_cxx[0]] \
+                         + cxx.linker_so[2:]
+     else:
+-        cxx.linker_so = [cxx.compiler_cxx[0]] + cxx.linker_so[1:]
++        cxx.linker_so = cxx.compiler_cxx + cxx.linker_so[1:]
+     return cxx
+ 
+ replace_method(CCompiler, 'cxx_compiler', CCompiler_cxx_compiler)

pythonforandroid/recipes/numpy/patches/do_not_use_system_libs.patch

@@ -0,0 +1,13 @@
+diff --git a/numpy/core/src/common/numpyos.c b/numpy/core/src/common/numpyos.c
+index d60b1ca..8972144 100644
+--- a/numpy/core/src/common/numpyos.c
++++ b/numpy/core/src/common/numpyos.c
+@@ -20,7 +20,7 @@
+      * the defines from xlocale.h are included in locale.h on some systems;
+      * see gh-8367
+      */
+-    #include <xlocale.h>
++    #include <locale.h>
+ #endif
+ #endif
+ 

pythonforandroid/recipes/numpy/patches/hostnumpy-xlocale.patch

@@ -0,0 +1,28 @@
+diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py
+index fc7018a..7b514bc 100644
+--- a/numpy/distutils/system_info.py
++++ b/numpy/distutils/system_info.py
+@@ -340,10 +340,10 @@ if os.path.join(sys.prefix, 'lib') not in default_lib_dirs:
+     default_include_dirs.append(os.path.join(sys.prefix, 'include'))
+     default_src_dirs.append(os.path.join(sys.prefix, 'src'))
+ 
+-default_lib_dirs = [_m for _m in default_lib_dirs if os.path.isdir(_m)]
+-default_runtime_dirs = [_m for _m in default_runtime_dirs if os.path.isdir(_m)]
+-default_include_dirs = [_m for _m in default_include_dirs if os.path.isdir(_m)]
+-default_src_dirs = [_m for _m in default_src_dirs if os.path.isdir(_m)]
++default_lib_dirs = [] #[_m for _m in default_lib_dirs if os.path.isdir(_m)]
++default_runtime_dirs =[] # [_m for _m in default_runtime_dirs if os.path.isdir(_m)]
++default_include_dirs =[] # [_m for _m in default_include_dirs if os.path.isdir(_m)]
++default_src_dirs =[] # [_m for _m in default_src_dirs if os.path.isdir(_m)]
+ 
+ so_ext = get_shared_lib_extension()
+ 
+@@ -814,7 +814,7 @@ class system_info(object):
+         path = self.get_paths(self.section, key)
+         if path == ['']:
+             path = []
+-        return path
++        return []
+ 
+     def get_include_dirs(self, key='include_dirs'):
+         return self.get_paths(self.section, key)

pythonforandroid/recipes/numpy/patches/remove-default-paths.patch

@@ -0,0 +1,49 @@
+from pythonforandroid.recipe import CompiledComponentsPythonRecipe, Recipe
+from multiprocessing import cpu_count
+from os.path import join
+
+
+class ScipyRecipe(CompiledComponentsPythonRecipe):
+
+    version = '1.5.4'
+    url = f'https://github.com/scipy/scipy/releases/download/v{version}/scipy-{version}.zip'
+    site_packages_name = 'scipy'
+    depends = ['setuptools', 'cython', 'numpy', 'lapack']
+    call_hostpython_via_targetpython = False
+
+    def build_compiled_components(self, arch):
+        self.setup_extra_args = ['-j', str(cpu_count())]
+        super().build_compiled_components(arch)
+        self.setup_extra_args = []
+
+    def rebuild_compiled_components(self, arch, env):
+        self.setup_extra_args = ['-j', str(cpu_count())]
+        super().rebuild_compiled_components(arch, env)
+        self.setup_extra_args = []
+
+    def get_recipe_env(self, arch):
+        env = super().get_recipe_env(arch)
+
+        GCC_VER = '4.9'
+        HOST = 'linux-x86_64'
+        LIB = 'lib64'
+
+        prefix = env['TOOLCHAIN_PREFIX']
+        lapack_dir = join(Recipe.get_recipe('lapack', self.ctx).get_build_dir(arch.arch), 'build', 'install')
+        sysroot = f"{self.ctx.ndk_dir}/platforms/{env['NDK_API']}/{arch.platform_dir}"
+        sysroot_include = f'{self.ctx.ndk_dir}/toolchains/llvm/prebuilt/{HOST}/sysroot/usr/include'
+        libgfortran = f'{self.ctx.ndk_dir}/toolchains/{prefix}-{GCC_VER}/prebuilt/{HOST}/{prefix}/{LIB}'
+        numpylib = self.ctx.get_python_install_dir() + '/numpy/core/lib'
+        LDSHARED_opts = env['LDSHARED'].split('clang')[1]
+
+        env['LAPACK'] = f'{lapack_dir}/lib'
+        env['BLAS'] = env['LAPACK']
+        env['F90'] = f'{prefix}-gfortran'
+        env['CXX'] += f' -Wl,-l{self.stl_lib_name} -Wl,-L{self.get_stl_lib_dir(arch)}'
+        env['CPPFLAGS'] += f' --sysroot={sysroot} -I{sysroot_include}/c++/v1 -I{sysroot_include}'
+        env['LDSHARED'] = 'clang'
+        env['LDFLAGS'] += f' {LDSHARED_opts} --sysroot={sysroot} -L{libgfortran} -L{numpylib}'
+        return env
+
+
+recipe = ScipyRecipe()