diff --git a/ci/constants.py b/ci/constants.py index 8b79bb5e54..8d62c83feb 100644 --- a/ci/constants.py +++ b/ci/constants.py @@ -26,6 +26,8 @@ class TargetPython(Enum): # mpmath package with a version >= 0.19 required 'sympy', 'vlc', + # need extra gfortran NDK system add-on + 'lapack', 'scipy', ]) BROKEN_RECIPES = { diff --git a/pythonforandroid/recipes/lapack/__init__.py b/pythonforandroid/recipes/lapack/__init__.py new file mode 100644 index 0000000000..dc11672e0b --- /dev/null +++ b/pythonforandroid/recipes/lapack/__init__.py @@ -0,0 +1,53 @@ +from pythonforandroid.recipe import Recipe +from pythonforandroid.logger import shprint +from pythonforandroid.util import current_directory, ensure_dir, BuildInterruptingException +from multiprocessing import cpu_count +from os.path import join +import sh + + +class LapackRecipe(Recipe): + + name = 'lapack' + version = 'v3.9.0' + url = 'https://github.com/Reference-LAPACK/lapack/archive/{version}.tar.gz' + libdir = 'build/install/lib' + built_libraries = {'libblas.so': libdir, 'liblapack.so': libdir, 'libcblas.so': libdir} + need_stl_shared = True + + def get_recipe_env(self, arch): + env = super().get_recipe_env(arch) + sysroot = f"{self.ctx.ndk_dir}/platforms/{env['NDK_API']}/{arch.platform_dir}" + FC = f"{env['TOOLCHAIN_PREFIX']}-gfortran" + env['FC'] = f'{FC} --sysroot={sysroot}' + if sh.which(FC) is None: + raise BuildInterruptingException(f"{FC} not found. See https://github.com/mzakharo/android-gfortran") + return env + + def build_arch(self, arch): + source_dir = self.get_build_dir(arch.arch) + build_target = join(source_dir, 'build') + install_target = join(build_target, 'install') + + ensure_dir(build_target) + with current_directory(build_target): + env = self.get_recipe_env(arch) + shprint(sh.rm, '-rf', 'CMakeFiles/', 'CMakeCache.txt', _env=env) + shprint(sh.cmake, source_dir, + '-DCMAKE_SYSTEM_NAME=Android', + '-DCMAKE_POSITION_INDEPENDENT_CODE=1', + '-DCMAKE_ANDROID_ARCH_ABI={arch}'.format(arch=arch.arch), + '-DCMAKE_ANDROID_NDK=' + self.ctx.ndk_dir, + '-DCMAKE_BUILD_TYPE=Release', + '-DCMAKE_INSTALL_PREFIX={}'.format(install_target), + '-DANDROID_ABI={arch}'.format(arch=arch.arch), + '-DANDROID_ARM_NEON=ON', + '-DENABLE_NEON=ON', + '-DCBLAS=ON', + '-DBUILD_SHARED_LIBS=ON', + _env=env) + shprint(sh.make, '-j' + str(cpu_count()), _env=env) + shprint(sh.make, 'install', _env=env) + + +recipe = LapackRecipe() diff --git a/pythonforandroid/recipes/numpy/__init__.py b/pythonforandroid/recipes/numpy/__init__.py index 334886f792..6b48e355ca 100644 --- a/pythonforandroid/recipes/numpy/__init__.py +++ b/pythonforandroid/recipes/numpy/__init__.py @@ -1,6 +1,10 @@ from pythonforandroid.recipe import CompiledComponentsPythonRecipe +from pythonforandroid.logger import shprint, info +from pythonforandroid.util import current_directory from multiprocessing import cpu_count from os.path import join +import glob +import sh class NumpyRecipe(CompiledComponentsPythonRecipe): @@ -9,23 +13,47 @@ class NumpyRecipe(CompiledComponentsPythonRecipe): url = 'https://pypi.python.org/packages/source/n/numpy/numpy-{version}.zip' site_packages_name = 'numpy' depends = ['setuptools', 'cython'] + install_in_hostpython = True + call_hostpython_via_targetpython = False patches = [ + join('patches', 'hostnumpy-xlocale.patch'), + join('patches', 'remove-default-paths.patch'), join('patches', 'add_libm_explicitly_to_build.patch'), - join('patches', 'do_not_use_system_libs.patch'), - join('patches', 'remove_unittest_call.patch'), + join('patches', 'compiler_cxx_fix.patch'), ] - call_hostpython_via_targetpython = False + def _build_compiled_components(self, arch): + info('Building compiled components in {}'.format(self.name)) + + env = self.get_recipe_env(arch) + with current_directory(self.get_build_dir(arch.arch)): + hostpython = sh.Command(self.real_hostpython_location) + if self.install_in_hostpython: + shprint(hostpython, 'setup.py', 'clean', '--all', '--force', _env=env) + hostpython = sh.Command(self.hostpython_location) + shprint(hostpython, 'setup.py', self.build_cmd, '-v', + _env=env, *self.setup_extra_args) + build_dir = glob.glob('build/lib.*')[0] + shprint(sh.find, build_dir, '-name', '"*.o"', '-exec', + env['STRIP'], '{}', ';', _env=env) + + def _rebuild_compiled_components(self, arch, env): + info('Rebuilding compiled components in {}'.format(self.name)) + + hostpython = sh.Command(self.real_hostpython_location) + shprint(hostpython, 'setup.py', 'clean', '--all', '--force', _env=env) + shprint(hostpython, 'setup.py', self.build_cmd, '-v', _env=env, + *self.setup_extra_args) def build_compiled_components(self, arch): self.setup_extra_args = ['-j', str(cpu_count())] - super().build_compiled_components(arch) + self._build_compiled_components(arch) self.setup_extra_args = [] def rebuild_compiled_components(self, arch, env): self.setup_extra_args = ['-j', str(cpu_count())] - super().rebuild_compiled_components(arch, env) + self._rebuild_compiled_components(arch, env) self.setup_extra_args = [] diff --git a/pythonforandroid/recipes/numpy/patches/compiler_cxx_fix.patch b/pythonforandroid/recipes/numpy/patches/compiler_cxx_fix.patch new file mode 100644 index 0000000000..fc2a557b32 --- /dev/null +++ b/pythonforandroid/recipes/numpy/patches/compiler_cxx_fix.patch @@ -0,0 +1,20 @@ +diff --git a/numpy/distutils/ccompiler.py b/numpy/distutils/ccompiler.py +index 6438790..a5f1527 100644 +--- a/numpy/distutils/ccompiler.py ++++ b/numpy/distutils/ccompiler.py +@@ -686,13 +686,13 @@ def CCompiler_cxx_compiler(self): + return self + + cxx = copy(self) +- cxx.compiler_so = [cxx.compiler_cxx[0]] + cxx.compiler_so[1:] ++ cxx.compiler_so = cxx.compiler_cxx + cxx.compiler_so[1:] + if sys.platform.startswith('aix') and 'ld_so_aix' in cxx.linker_so[0]: + # AIX needs the ld_so_aix script included with Python + cxx.linker_so = [cxx.linker_so[0], cxx.compiler_cxx[0]] \ + + cxx.linker_so[2:] + else: +- cxx.linker_so = [cxx.compiler_cxx[0]] + cxx.linker_so[1:] ++ cxx.linker_so = cxx.compiler_cxx + cxx.linker_so[1:] + return cxx + + replace_method(CCompiler, 'cxx_compiler', CCompiler_cxx_compiler) diff --git a/pythonforandroid/recipes/numpy/patches/do_not_use_system_libs.patch b/pythonforandroid/recipes/numpy/patches/do_not_use_system_libs.patch deleted file mode 100644 index 2fc1b8db12..0000000000 --- a/pythonforandroid/recipes/numpy/patches/do_not_use_system_libs.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py -index 806f4f7..0d51cfa 100644 ---- a/numpy/distutils/system_info.py -+++ b/numpy/distutils/system_info.py -@@ -823,6 +823,7 @@ class system_info(object): - return self.get_paths(self.section, key) - - def get_libs(self, key, default): -+ return [] - try: - libs = self.cp.get(self.section, key) - except NoOptionError: diff --git a/pythonforandroid/recipes/numpy/patches/hostnumpy-xlocale.patch b/pythonforandroid/recipes/numpy/patches/hostnumpy-xlocale.patch new file mode 100644 index 0000000000..511b06dea0 --- /dev/null +++ b/pythonforandroid/recipes/numpy/patches/hostnumpy-xlocale.patch @@ -0,0 +1,13 @@ +diff --git a/numpy/core/src/common/numpyos.c b/numpy/core/src/common/numpyos.c +index d60b1ca..8972144 100644 +--- a/numpy/core/src/common/numpyos.c ++++ b/numpy/core/src/common/numpyos.c +@@ -20,7 +20,7 @@ + * the defines from xlocale.h are included in locale.h on some systems; + * see gh-8367 + */ +- #include ++ #include + #endif + #endif + diff --git a/pythonforandroid/recipes/numpy/patches/remove-default-paths.patch b/pythonforandroid/recipes/numpy/patches/remove-default-paths.patch new file mode 100644 index 0000000000..3581f0f9ed --- /dev/null +++ b/pythonforandroid/recipes/numpy/patches/remove-default-paths.patch @@ -0,0 +1,28 @@ +diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py +index fc7018a..7b514bc 100644 +--- a/numpy/distutils/system_info.py ++++ b/numpy/distutils/system_info.py +@@ -340,10 +340,10 @@ if os.path.join(sys.prefix, 'lib') not in default_lib_dirs: + default_include_dirs.append(os.path.join(sys.prefix, 'include')) + default_src_dirs.append(os.path.join(sys.prefix, 'src')) + +-default_lib_dirs = [_m for _m in default_lib_dirs if os.path.isdir(_m)] +-default_runtime_dirs = [_m for _m in default_runtime_dirs if os.path.isdir(_m)] +-default_include_dirs = [_m for _m in default_include_dirs if os.path.isdir(_m)] +-default_src_dirs = [_m for _m in default_src_dirs if os.path.isdir(_m)] ++default_lib_dirs = [] #[_m for _m in default_lib_dirs if os.path.isdir(_m)] ++default_runtime_dirs =[] # [_m for _m in default_runtime_dirs if os.path.isdir(_m)] ++default_include_dirs =[] # [_m for _m in default_include_dirs if os.path.isdir(_m)] ++default_src_dirs =[] # [_m for _m in default_src_dirs if os.path.isdir(_m)] + + so_ext = get_shared_lib_extension() + +@@ -814,7 +814,7 @@ class system_info(object): + path = self.get_paths(self.section, key) + if path == ['']: + path = [] +- return path ++ return [] + + def get_include_dirs(self, key='include_dirs'): + return self.get_paths(self.section, key) diff --git a/pythonforandroid/recipes/numpy/patches/remove_unittest_call.patch b/pythonforandroid/recipes/numpy/patches/remove_unittest_call.patch deleted file mode 100644 index e19dc0f814..0000000000 --- a/pythonforandroid/recipes/numpy/patches/remove_unittest_call.patch +++ /dev/null @@ -1,30 +0,0 @@ -diff --git a/numpy/testing/__init__.py b/numpy/testing/__init__.py -index a8bd4fc..6b01fa6 100644 ---- a/numpy/testing/__init__.py -+++ b/numpy/testing/__init__.py -@@ -5,18 +5,11 @@ in a single location, so that test scripts can just import it and work right - away. - - """ --from __future__ import division, absolute_import, print_function - --from unittest import TestCase -- --from ._private.utils import * --from ._private import decorators as dec --from ._private.nosetester import ( -- run_module_suite, NoseTester as Tester -- ) -- --__all__ = _private.utils.__all__ + ['TestCase', 'run_module_suite'] -- --from numpy._pytesttester import PytestTester --test = PytestTester(__name__) --del PytestTester -+# fake tester, android don't have unittest -+class Tester(object): -+ def test(self, *args, **kwargs): -+ pass -+ def bench(self, *args, **kwargs): -+ pass -+test = Tester().test diff --git a/pythonforandroid/recipes/scipy/__init__.py b/pythonforandroid/recipes/scipy/__init__.py new file mode 100644 index 0000000000..8753f3a365 --- /dev/null +++ b/pythonforandroid/recipes/scipy/__init__.py @@ -0,0 +1,49 @@ +from pythonforandroid.recipe import CompiledComponentsPythonRecipe, Recipe +from multiprocessing import cpu_count +from os.path import join + + +class ScipyRecipe(CompiledComponentsPythonRecipe): + + version = '1.5.4' + url = f'https://github.com/scipy/scipy/releases/download/v{version}/scipy-{version}.zip' + site_packages_name = 'scipy' + depends = ['setuptools', 'cython', 'numpy', 'lapack'] + call_hostpython_via_targetpython = False + + def build_compiled_components(self, arch): + self.setup_extra_args = ['-j', str(cpu_count())] + super().build_compiled_components(arch) + self.setup_extra_args = [] + + def rebuild_compiled_components(self, arch, env): + self.setup_extra_args = ['-j', str(cpu_count())] + super().rebuild_compiled_components(arch, env) + self.setup_extra_args = [] + + def get_recipe_env(self, arch): + env = super().get_recipe_env(arch) + + GCC_VER = '4.9' + HOST = 'linux-x86_64' + LIB = 'lib64' + + prefix = env['TOOLCHAIN_PREFIX'] + lapack_dir = join(Recipe.get_recipe('lapack', self.ctx).get_build_dir(arch.arch), 'build', 'install') + sysroot = f"{self.ctx.ndk_dir}/platforms/{env['NDK_API']}/{arch.platform_dir}" + sysroot_include = f'{self.ctx.ndk_dir}/toolchains/llvm/prebuilt/{HOST}/sysroot/usr/include' + libgfortran = f'{self.ctx.ndk_dir}/toolchains/{prefix}-{GCC_VER}/prebuilt/{HOST}/{prefix}/{LIB}' + numpylib = self.ctx.get_python_install_dir() + '/numpy/core/lib' + LDSHARED_opts = env['LDSHARED'].split('clang')[1] + + env['LAPACK'] = f'{lapack_dir}/lib' + env['BLAS'] = env['LAPACK'] + env['F90'] = f'{prefix}-gfortran' + env['CXX'] += f' -Wl,-l{self.stl_lib_name} -Wl,-L{self.get_stl_lib_dir(arch)}' + env['CPPFLAGS'] += f' --sysroot={sysroot} -I{sysroot_include}/c++/v1 -I{sysroot_include}' + env['LDSHARED'] = 'clang' + env['LDFLAGS'] += f' {LDSHARED_opts} --sysroot={sysroot} -L{libgfortran} -L{numpylib}' + return env + + +recipe = ScipyRecipe()