From f32c86af51ae62d719a2b9c0b6d344293120b39c Mon Sep 17 00:00:00 2001 From: Mikhail Zakharov Date: Fri, 14 Jan 2022 07:28:33 -0500 Subject: [PATCH 1/2] add scikit-learn recipe and dependencies --- ci/constants.py | 2 +- pythonforandroid/recipes/joblib/__init__.py | 14 +++++++ .../recipes/joblib/multiprocessing.patch | 32 +++++++++++++++ .../recipes/scikit-learn/__init__.py | 40 +++++++++++++++++++ .../recipes/scikit-learn/cross-compile.patch | 30 ++++++++++++++ .../recipes/threadpoolctl/__init__.py | 14 +++++++ .../recipes/threadpoolctl/docstring.patch | 13 ++++++ .../recipes/threadpoolctl/setuptools.patch | 15 +++++++ 8 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 pythonforandroid/recipes/joblib/__init__.py create mode 100644 pythonforandroid/recipes/joblib/multiprocessing.patch create mode 100644 pythonforandroid/recipes/scikit-learn/__init__.py create mode 100644 pythonforandroid/recipes/scikit-learn/cross-compile.patch create mode 100644 pythonforandroid/recipes/threadpoolctl/__init__.py create mode 100644 pythonforandroid/recipes/threadpoolctl/docstring.patch create mode 100644 pythonforandroid/recipes/threadpoolctl/setuptools.patch diff --git a/ci/constants.py b/ci/constants.py index f794bc68b9..66a04327aa 100644 --- a/ci/constants.py +++ b/ci/constants.py @@ -27,7 +27,7 @@ class TargetPython(Enum): 'sympy', 'vlc', # need extra gfortran NDK system add-on - 'lapack', 'scipy', + 'lapack', 'scipy', 'scikit-learn', # Outdated and there's a chance that is now useless. 'zope_interface', # Requires zope_interface, which is broken. diff --git a/pythonforandroid/recipes/joblib/__init__.py b/pythonforandroid/recipes/joblib/__init__.py new file mode 100644 index 0000000000..19dc7952c6 --- /dev/null +++ b/pythonforandroid/recipes/joblib/__init__.py @@ -0,0 +1,14 @@ +from pythonforandroid.recipe import PythonRecipe + + +class ThisRecipe(PythonRecipe): + org = 'joblib' + name = 'joblib' + version = '0.17.0' + url = f'https://github.com/{org}/{name}/archive/{version}.zip' + depends = ['setuptools'] + call_hostpython_via_targetpython = False + patches = ['multiprocessing.patch'] + + +recipe = ThisRecipe() diff --git a/pythonforandroid/recipes/joblib/multiprocessing.patch b/pythonforandroid/recipes/joblib/multiprocessing.patch new file mode 100644 index 0000000000..defecd1194 --- /dev/null +++ b/pythonforandroid/recipes/joblib/multiprocessing.patch @@ -0,0 +1,32 @@ +diff --git a/joblib/externals/loky/backend/__init__.py b/joblib/externals/loky/backend/__init__.py +index a65ce0e..9efdb46 100644 +--- a/joblib/externals/loky/backend/__init__.py ++++ b/joblib/externals/loky/backend/__init__.py +@@ -10,7 +10,7 @@ if sys.version_info > (3, 4): + return name + + # monkey patch the name creation for multiprocessing +- from multiprocessing import synchronize +- synchronize.SemLock._make_name = staticmethod(_make_name) ++ #from multiprocessing import synchronize ++ #synchronize.SemLock._make_name = staticmethod(_make_name) + + __all__ = ["get_context"] +diff --git a/joblib/externals/loky/backend/queues.py b/joblib/externals/loky/backend/queues.py +index 62735db..156e235 100644 +--- a/joblib/externals/loky/backend/queues.py ++++ b/joblib/externals/loky/backend/queues.py +@@ -9,6 +9,8 @@ + # * Add some custom reducers for the Queues/SimpleQueue to tweak the + # pickling process. (overload Queue._feed/SimpleQueue.put) + # ++from queue import Queue, SimpleQueue, Full ++''' + import os + import sys + import errno +@@ -245,3 +247,4 @@ class SimpleQueue(mp_SimpleQueue): + else: + with self._wlock: + self._writer.send_bytes(obj) ++''' diff --git a/pythonforandroid/recipes/scikit-learn/__init__.py b/pythonforandroid/recipes/scikit-learn/__init__.py new file mode 100644 index 0000000000..f7cf57a1b1 --- /dev/null +++ b/pythonforandroid/recipes/scikit-learn/__init__.py @@ -0,0 +1,40 @@ +from pythonforandroid.recipe import CompiledComponentsPythonRecipe, Recipe +from multiprocessing import cpu_count + + +class ThisRecipe(CompiledComponentsPythonRecipe): + + site_packages_name = 'scikit-learn' + version = '0.23.2' + url = f'https://github.com/{site_packages_name}/{site_packages_name}/archive/{version}.zip' + depends = ['setuptools', 'scipy', 'joblib', 'threadpoolctl'] + call_hostpython_via_targetpython = False + need_stl_shared = True + patches = ['cross-compile.patch'] + + def build_compiled_components(self, arch): + self.setup_extra_args = ['-j', str(cpu_count())] + super().build_compiled_components(arch) + self.setup_extra_args = [] + + def rebuild_compiled_components(self, arch, env): + self.setup_extra_args = ['-j', str(cpu_count())] + super().rebuild_compiled_components(arch, env) + self.setup_extra_args = [] + + def strip_ccache(self, env): + for key, value in env.items(): + parts = value.split(' ') + if 'ccache' in parts[0]: + env[key] = ' '.join(parts[1:]) + + def get_recipe_env(self, arch): + env = super().get_recipe_env(arch) + self.strip_ccache(env) + scipy_build_dir = Recipe.get_recipe('scipy', self.ctx).get_build_dir(arch.arch) + env['PYTHONPATH'] += f':{scipy_build_dir}' + env['CXX'] += f' -Wl,-l{self.stl_lib_name}' + return env + + +recipe = ThisRecipe() diff --git a/pythonforandroid/recipes/scikit-learn/cross-compile.patch b/pythonforandroid/recipes/scikit-learn/cross-compile.patch new file mode 100644 index 0000000000..e747edcdac --- /dev/null +++ b/pythonforandroid/recipes/scikit-learn/cross-compile.patch @@ -0,0 +1,30 @@ +diff --git a/setup.py b/setup.py +index 90162b6..946af48 100755 +--- a/setup.py ++++ b/setup.py +@@ -291,7 +291,7 @@ def setup_package(): + + check_package_status('numpy', NUMPY_MIN_VERSION) + +- check_package_status('scipy', SCIPY_MIN_VERSION) ++ #check_package_status('scipy', SCIPY_MIN_VERSION) + + from numpy.distutils.core import setup + +diff --git a/sklearn/_build_utils/pre_build_helpers.py b/sklearn/_build_utils/pre_build_helpers.py +index bc3d832..fe1819e 100644 +--- a/sklearn/_build_utils/pre_build_helpers.py ++++ b/sklearn/_build_utils/pre_build_helpers.py +@@ -48,8 +48,10 @@ def compile_test_program(code, extra_preargs=[], extra_postargs=[]): + + # Run test program + # will raise a CalledProcessError if return code was non-zero +- output = subprocess.check_output('./test_program') +- output = output.decode(sys.stdout.encoding or 'utf-8').splitlines() ++ assert os.path.exists('./test_program') ++ output = ['nthreads=4'] * 4 ++ #output = subprocess.check_output('./test_program') ++ #output = output.decode(sys.stdout.encoding or 'utf-8').splitlines() + except Exception: + raise + finally: diff --git a/pythonforandroid/recipes/threadpoolctl/__init__.py b/pythonforandroid/recipes/threadpoolctl/__init__.py new file mode 100644 index 0000000000..bedccbac5e --- /dev/null +++ b/pythonforandroid/recipes/threadpoolctl/__init__.py @@ -0,0 +1,14 @@ +from pythonforandroid.recipe import PythonRecipe + + +class ThisRecipe(PythonRecipe): + org = 'joblib' + name = 'threadpoolctl' + version = '2.1.0' + url = f'https://github.com/{org}/{name}/archive/{version}.zip' + depends = ['setuptools'] + call_hostpython_via_targetpython = False + patches = ['docstring.patch', 'setuptools.patch'] + + +recipe = ThisRecipe() diff --git a/pythonforandroid/recipes/threadpoolctl/docstring.patch b/pythonforandroid/recipes/threadpoolctl/docstring.patch new file mode 100644 index 0000000000..f5a6aee304 --- /dev/null +++ b/pythonforandroid/recipes/threadpoolctl/docstring.patch @@ -0,0 +1,13 @@ +diff --git a/threadpoolctl.py b/threadpoolctl.py +index b94d7a1..faf7a44 100644 +--- a/threadpoolctl.py ++++ b/threadpoolctl.py +@@ -98,7 +98,7 @@ _ALL_OPENMP_LIBRARIES = list( + + def _format_docstring(*args, **kwargs): + def decorator(o): +- o.__doc__ = o.__doc__.format(*args, **kwargs) ++ #o.__doc__ = o.__doc__.format(*args, **kwargs) + return o + + return decorator diff --git a/pythonforandroid/recipes/threadpoolctl/setuptools.patch b/pythonforandroid/recipes/threadpoolctl/setuptools.patch new file mode 100644 index 0000000000..10db2fd8a7 --- /dev/null +++ b/pythonforandroid/recipes/threadpoolctl/setuptools.patch @@ -0,0 +1,15 @@ +diff --git a/setup.py b/setup.py +new file mode 100755 +index 0000000..b18d2fd +--- /dev/null ++++ b/setup.py +@@ -0,0 +1,9 @@ ++from setuptools import setup ++import threadpoolctl ++ ++if __name__ == '__main__': ++ setup(name='threadpoolctl', ++ version=threadpoolctl.__version__, ++ py_modules = ['threadpoolctl'], ++ python_requires='>=3.6', ++ ) From 046771c210e2d802504d197d1cafd07091b2f93f Mon Sep 17 00:00:00 2001 From: Mikhail Zakharov Date: Mon, 21 Mar 2022 09:54:26 -0400 Subject: [PATCH 2/2] sklearn: follow recipe naming conventions --- pythonforandroid/recipes/joblib/__init__.py | 4 ++-- pythonforandroid/recipes/scikit-learn/__init__.py | 4 ++-- pythonforandroid/recipes/threadpoolctl/__init__.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pythonforandroid/recipes/joblib/__init__.py b/pythonforandroid/recipes/joblib/__init__.py index 19dc7952c6..c7ff6be68d 100644 --- a/pythonforandroid/recipes/joblib/__init__.py +++ b/pythonforandroid/recipes/joblib/__init__.py @@ -1,7 +1,7 @@ from pythonforandroid.recipe import PythonRecipe -class ThisRecipe(PythonRecipe): +class JoblibRecipe(PythonRecipe): org = 'joblib' name = 'joblib' version = '0.17.0' @@ -11,4 +11,4 @@ class ThisRecipe(PythonRecipe): patches = ['multiprocessing.patch'] -recipe = ThisRecipe() +recipe = JoblibRecipe() diff --git a/pythonforandroid/recipes/scikit-learn/__init__.py b/pythonforandroid/recipes/scikit-learn/__init__.py index f7cf57a1b1..c80b84f1f3 100644 --- a/pythonforandroid/recipes/scikit-learn/__init__.py +++ b/pythonforandroid/recipes/scikit-learn/__init__.py @@ -2,7 +2,7 @@ from multiprocessing import cpu_count -class ThisRecipe(CompiledComponentsPythonRecipe): +class ScikitlearnRecipe(CompiledComponentsPythonRecipe): site_packages_name = 'scikit-learn' version = '0.23.2' @@ -37,4 +37,4 @@ def get_recipe_env(self, arch): return env -recipe = ThisRecipe() +recipe = ScikitlearnRecipe() diff --git a/pythonforandroid/recipes/threadpoolctl/__init__.py b/pythonforandroid/recipes/threadpoolctl/__init__.py index bedccbac5e..6a561d5d9a 100644 --- a/pythonforandroid/recipes/threadpoolctl/__init__.py +++ b/pythonforandroid/recipes/threadpoolctl/__init__.py @@ -1,7 +1,7 @@ from pythonforandroid.recipe import PythonRecipe -class ThisRecipe(PythonRecipe): +class ThreadpoolctlRecipe(PythonRecipe): org = 'joblib' name = 'threadpoolctl' version = '2.1.0' @@ -11,4 +11,4 @@ class ThisRecipe(PythonRecipe): patches = ['docstring.patch', 'setuptools.patch'] -recipe = ThisRecipe() +recipe = ThreadpoolctlRecipe()