Skip to content

Commit b88e82b

Browse files
obfuskobfusk
committed
rework platform & sysroot usage for ndk >= r22
* add & use - ctx.ndk_standalone - ctx.ndk_sysroot - ctx.ndk_lib_dir - ctx.ndk_include_dir * patch openssl * mark some packages as broken for CI * add FIXME comments where testing/modification for ndk >= 22 is needed
1 parent aadcfc4 commit b88e82b

File tree

24 files changed

+135
-45
lines changed

24 files changed

+135
-45
lines changed

‎ci/constants.py

+4
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,10 @@ class TargetPython(Enum):
2828
'vlc',
2929
# need extra gfortran NDK system add-on
3030
'lapack', 'scipy',
31+
# 403 on https://jqueryui.com/resources/download/jquery-ui-1.12.1.zip
32+
'matplotlib',
33+
# requires kernel headers
34+
'evdev',
3135
])
3236

3337
BROKEN_RECIPES = {

‎pythonforandroid/archs.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ class Arch:
3030
common_cppflags = [
3131
'-DANDROID',
3232
'-D__ANDROID_API__={ctx.ndk_api}',
33-
'-I{ctx.ndk_dir}/sysroot/usr/include/{command_prefix}',
33+
'-I{ctx.ndk_sysroot}/usr/include/{command_prefix}',
3434
'-I{python_includes}',
3535
]
3636

‎pythonforandroid/build.py

+36-6
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,10 @@
1212
import shutil
1313
import subprocess
1414
from contextlib import suppress
15+
from distutils.version import LooseVersion
1516

1617
from pythonforandroid.util import (
17-
current_directory, ensure_dir,
18+
build_platform, current_directory, ensure_dir,
1819
BuildInterruptingException,
1920
)
2021
from pythonforandroid.logger import (info, warning, info_notify, info_main, shprint)
@@ -23,7 +24,7 @@
2324
from pythonforandroid.recipe import CythonRecipe, Recipe
2425
from pythonforandroid.recommendations import (
2526
check_ndk_version, check_target_api, check_ndk_api,
26-
RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API)
27+
read_ndk_version, RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API)
2728

2829

2930
def get_ndk_platform_dir(ndk_dir, ndk_api, arch):
@@ -40,6 +41,21 @@ def get_ndk_platform_dir(ndk_dir, ndk_api, arch):
4041
return ndk_platform, ndk_platform_dir_exists
4142

4243

44+
def get_ndk_standalone(ndk_dir):
45+
return join(ndk_dir, 'toolchains', 'llvm', 'prebuilt', build_platform)
46+
47+
48+
def get_ndk_sysroot(ndk_dir):
49+
sysroot = join(get_ndk_standalone(ndk_dir), 'sysroot')
50+
if not exists(sysroot):
51+
warning("sysroot doesn't exist: {}".format(sysroot))
52+
return sysroot
53+
54+
55+
def get_ndk_lib_dir(sysroot, ndk_api, arch):
56+
return join(sysroot, 'usr', 'lib', arch.command_prefix, str(ndk_api))
57+
58+
4359
def get_toolchain_versions(ndk_dir, arch):
4460
toolchain_versions = []
4561
toolchain_path_exists = True
@@ -115,6 +131,10 @@ class Context:
115131
ccache = None # whether to use ccache
116132

117133
ndk_platform = None # the ndk platform directory
134+
ndk_standalone = None # ndk >= r22 doesn't have platform/ & sysroot/
135+
ndk_sysroot = None
136+
ndk_lib_dir = None # usr/lib
137+
ndk_include_dir = None # usr/include
118138

119139
bootstrap = None
120140
bootstrap_build_dir = None
@@ -378,9 +398,19 @@ def prepare_build_environment(self,
378398
# This would need to be changed if supporting multiarch APKs
379399
arch = self.archs[0]
380400
toolchain_prefix = arch.toolchain_prefix
381-
self.ndk_platform, ndk_platform_dir_exists = get_ndk_platform_dir(
382-
self.ndk_dir, self.ndk_api, arch)
383-
ok = ok and ndk_platform_dir_exists
401+
ndk_version = read_ndk_version(ndk_dir)
402+
if ndk_version is None or ndk_version < LooseVersion('22'):
403+
self.ndk_platform, ndk_platform_dir_exists = get_ndk_platform_dir(
404+
self.ndk_dir, self.ndk_api, arch)
405+
ok = ok and ndk_platform_dir_exists
406+
self.ndk_sysroot = join(self.ndk_dir, 'sysroot')
407+
self.ndk_lib_dir = join(self.ndk_dir, 'usr', 'lib')
408+
else:
409+
self.ndk_standalone = get_ndk_standalone(self.ndk_dir)
410+
self.ndk_sysroot = get_ndk_sysroot(self.ndk_dir)
411+
self.ndk_lib_dir = get_ndk_lib_dir(self.ndk_sysroot, self.ndk_api, arch)
412+
ok = ok and exists(self.ndk_sysroot)
413+
self.ndk_include_dir = join(self.ndk_sysroot, 'usr', 'include')
384414

385415
py_platform = sys.platform
386416
if py_platform in ['linux2', 'linux3']:
@@ -878,7 +908,7 @@ def biglink(ctx, arch):
878908

879909
# Move to the directory containing crtstart_so.o and crtend_so.o
880910
# This is necessary with newer NDKs? A gcc bug?
881-
with current_directory(join(ctx.ndk_platform, 'usr', 'lib')):
911+
with current_directory(ctx.ndk_lib_dir):
882912
do_biglink(
883913
join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'),
884914
obj_dir.split(' '),

‎pythonforandroid/recipe.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -1131,7 +1131,10 @@ def get_recipe_env(self, arch, with_flags_in_cc=True):
11311131
env['LDSHARED'] = env['CC'] + ' -shared'
11321132
# shprint(sh.whereis, env['LDSHARED'], _env=env)
11331133
env['LIBLINK'] = 'NOTNONE'
1134-
env['NDKPLATFORM'] = self.ctx.ndk_platform
1134+
if self.ctx.ndk_standalone:
1135+
env['NDKPLATFORM'] = self.ctx.ndk_sysroot # FIXME
1136+
else:
1137+
env['NDKPLATFORM'] = self.ctx.ndk_platform
11351138
if self.ctx.copy_libs:
11361139
env['COPYLIBS'] = '1'
11371140

‎pythonforandroid/recipes/Pillow/__init__.py

+7-3
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,13 @@ class PillowRecipe(CompiledComponentsPythonRecipe):
3535
def get_recipe_env(self, arch=None, with_flags_in_cc=True):
3636
env = super().get_recipe_env(arch, with_flags_in_cc)
3737

38-
env['ANDROID_ROOT'] = join(self.ctx.ndk_platform, 'usr')
39-
ndk_lib_dir = join(self.ctx.ndk_platform, 'usr', 'lib')
40-
ndk_include_dir = join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include')
38+
if self.ctx.ndk_standalone:
39+
pass # FIXME
40+
else:
41+
env['ANDROID_ROOT'] = join(self.ctx.ndk_platform, 'usr')
42+
43+
ndk_lib_dir = self.ctx.ndk_lib_dir
44+
ndk_include_dir = self.ctx.ndk_include_dir
4145

4246
png = self.get_recipe('png', self.ctx)
4347
png_lib_dir = join(png.get_build_dir(arch.arch), '.libs')

‎pythonforandroid/recipes/audiostream/__init__.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,10 @@ def get_recipe_env(self, arch):
2525
jni_path=join(self.ctx.bootstrap.build_dir, 'jni'),
2626
sdl_include=sdl_include,
2727
sdl_mixer_include=sdl_mixer_include)
28-
env['NDKPLATFORM'] = self.ctx.ndk_platform
28+
if self.ctx.ndk_standalone:
29+
env['NDKPLATFORM'] = self.ctx.ndk_sysroot # FIXME
30+
else:
31+
env['NDKPLATFORM'] = self.ctx.ndk_platform
2932
env['LIBLINK'] = 'NOTNONE' # Hacky fix. Needed by audiostream setup.py
3033
return env
3134

‎pythonforandroid/recipes/cffi/__init__.py

+1-3
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,7 @@ def get_recipe_env(self, arch=None):
3535
self.ctx.get_libs_dir(arch.arch))
3636
env['LDFLAGS'] += ' -L{}'.format(os.path.join(self.ctx.bootstrap.build_dir, 'libs', arch.arch))
3737
# required for libc and libdl
38-
ndk_dir = self.ctx.ndk_platform
39-
ndk_lib_dir = os.path.join(ndk_dir, 'usr', 'lib')
40-
env['LDFLAGS'] += ' -L{}'.format(ndk_lib_dir)
38+
env['LDFLAGS'] += ' -L{}'.format(self.ctx.ndk_lib_dir)
4139
env['PYTHONPATH'] = ':'.join([
4240
self.ctx.get_site_packages_dir(),
4341
env['BUILDLIB_PATH'],

‎pythonforandroid/recipes/evdev/__init__.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,10 @@ class EvdevRecipe(CompiledComponentsPythonRecipe):
1818

1919
def get_recipe_env(self, arch=None):
2020
env = super().get_recipe_env(arch)
21-
env['NDKPLATFORM'] = self.ctx.ndk_platform
21+
if self.ctx.ndk_standalone:
22+
env['NDKPLATFORM'] = self.ctx.ndk_sysroot # FIXME
23+
else:
24+
env['NDKPLATFORM'] = self.ctx.ndk_platform
2225
return env
2326

2427

‎pythonforandroid/recipes/ffmpeg/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ def build_arch(self, arch):
122122
'--strip={}strip'.format(cross_prefix),
123123
'--sysroot={}'.format(join(self.ctx.ndk_dir, 'toolchains',
124124
'llvm', 'prebuilt', 'linux-x86_64',
125-
'sysroot')),
125+
'sysroot')), # FIXME
126126
'--enable-neon',
127127
'--prefix={}'.format(realpath('.')),
128128
]

‎pythonforandroid/recipes/freetype/__init__.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -46,14 +46,14 @@ def get_recipe_env(self, arch=None, with_harfbuzz=False):
4646
)
4747
)
4848

49-
# android's zlib support
50-
zlib_lib_path = join(self.ctx.ndk_platform, 'usr', 'lib')
51-
zlib_includes = join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include')
52-
5349
def add_flag_if_not_added(flag, env_key):
5450
if flag not in env[env_key]:
5551
env[env_key] += flag
5652

53+
# android's zlib support
54+
zlib_lib_path = self.ctx.ndk_lib_dir
55+
zlib_includes = self.ctx.ndk_include_dir
56+
5757
add_flag_if_not_added(' -I' + zlib_includes, 'CFLAGS')
5858
add_flag_if_not_added(' -L' + zlib_lib_path, 'LDFLAGS')
5959
add_flag_if_not_added(' -lz', 'LDLIBS')

‎pythonforandroid/recipes/jpeg/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ def build_arch(self, arch):
3333
'-DCMAKE_SYSTEM_PROCESSOR={cpu}'.format(cpu='arm'),
3434
'-DCMAKE_POSITION_INDEPENDENT_CODE=1',
3535
'-DCMAKE_ANDROID_ARCH_ABI={arch}'.format(arch=arch.arch),
36-
'-DCMAKE_ANDROID_NDK=' + self.ctx.ndk_dir,
36+
'-DCMAKE_ANDROID_NDK=' + self.ctx.ndk_dir, # FIXME
3737
'-DCMAKE_C_COMPILER={cc}'.format(cc=arch.get_clang_exe()),
3838
'-DCMAKE_CXX_COMPILER={cc_plus}'.format(
3939
cc_plus=arch.get_clang_exe(plus_plus=True)),

‎pythonforandroid/recipes/libogg/__init__.py

+5-4
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,11 @@ class OggRecipe(Recipe):
1111
def build_arch(self, arch):
1212
with current_directory(self.get_build_dir(arch.arch)):
1313
env = self.get_recipe_env(arch)
14-
flags = [
15-
'--with-sysroot=' + self.ctx.ndk_platform,
16-
'--host=' + arch.toolchain_prefix,
17-
]
14+
flags = ['--host=' + arch.toolchain_prefix]
15+
if self.ctx.ndk_standalone:
16+
flags.append('--with-sysroot=' + self.ctx.ndk_sysroot) # FIXME
17+
else:
18+
flags.append('--with-sysroot=' + self.ctx.ndk_platform)
1819
configure = sh.Command('./configure')
1920
shprint(configure, *flags, _env=env)
2021
shprint(sh.make, _env=env)

‎pythonforandroid/recipes/librt/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ class LibRt(Recipe):
2020

2121
@property
2222
def libc_path(self):
23-
return join(self.ctx.ndk_platform, 'usr', 'lib', 'libc')
23+
return join(self.ctx.ndk_lib_dir, 'libc') # FIXME
2424

2525
def build_arch(self, arch):
2626
# Create a temporary folder to add to link path with a fake librt.so:

‎pythonforandroid/recipes/libvorbis/__init__.py

+5-4
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,11 @@ def get_recipe_env(self, arch=None):
2020
def build_arch(self, arch):
2121
with current_directory(self.get_build_dir(arch.arch)):
2222
env = self.get_recipe_env(arch)
23-
flags = [
24-
'--with-sysroot=' + self.ctx.ndk_platform,
25-
'--host=' + arch.toolchain_prefix,
26-
]
23+
flags = ['--host=' + arch.toolchain_prefix]
24+
if self.ctx.ndk_standalone:
25+
flags.append('--with-sysroot=' + self.ctx.ndk_sysroot) # FIXME
26+
else:
27+
flags.append('--with-sysroot=' + self.ctx.ndk_platform)
2728
configure = sh.Command('./configure')
2829
shprint(configure, *flags, _env=env)
2930
shprint(sh.make, _env=env)

‎pythonforandroid/recipes/lxml/__init__.py

+2-4
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,8 @@ def get_recipe_env(self, arch):
5151
env['LIBS'] += ' -lxml2'
5252

5353
# android's ndk flags
54-
ndk_lib_dir = join(self.ctx.ndk_platform, 'usr', 'lib')
55-
ndk_include_dir = join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include')
56-
cflags += ' -I' + ndk_include_dir
57-
env['LDFLAGS'] += ' -L' + ndk_lib_dir
54+
cflags += ' -I' + self.ctx.ndk_include_dir
55+
env['LDFLAGS'] += ' -L' + self.ctx.ndk_lib_dir
5856
env['LIBS'] += ' -lz -lm -lc'
5957

6058
if cflags not in env['CFLAGS']:

‎pythonforandroid/recipes/matplotlib/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ def prebuild_arch(self, arch):
9898

9999
with open(join(self.get_build_dir(arch), 'setup.cfg'), 'w') as fileh:
100100
fileh.write(setup_cfg.format(
101-
ndk_sysroot_usr=join(self.ctx.ndk_dir, 'sysroot', 'usr')))
101+
ndk_sysroot_usr=join(self.ctx.ndk_sysroot, 'usr'))) # FIXME
102102

103103
self.generate_libraries_pc_files(arch)
104104
self.download_web_backend_dependencies(arch)

‎pythonforandroid/recipes/opencv/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ def get_lib_dir(self, arch):
3939

4040
def get_recipe_env(self, arch):
4141
env = super().get_recipe_env(arch)
42-
env['ANDROID_NDK'] = self.ctx.ndk_dir
42+
env['ANDROID_NDK'] = self.ctx.ndk_dir # FIXME
4343
env['ANDROID_SDK'] = self.ctx.sdk_dir
4444
return env
4545

‎pythonforandroid/recipes/openssl/__init__.py

+7-1
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,11 @@ def get_recipe_env(self, arch=None):
9696
env = super().get_recipe_env(arch)
9797
env['OPENSSL_VERSION'] = self.version
9898
env['MAKE'] = 'make' # This removes the '-j5', which isn't safe
99-
env['ANDROID_NDK'] = self.ctx.ndk_dir
99+
if self.ctx.ndk_standalone:
100+
env['ANDROID_NDK_HOME'] = self.ctx.ndk_standalone
101+
env['CC'] = 'clang'
102+
else:
103+
env['ANDROID_NDK_HOME'] = self.ctx.ndk_dir
100104
return env
101105

102106
def select_build_arch(self, arch):
@@ -127,6 +131,8 @@ def build_arch(self, arch):
127131
buildarch,
128132
'-D__ANDROID_API__={}'.format(self.ctx.ndk_api),
129133
]
134+
if self.ctx.ndk_standalone:
135+
self.apply_patch('standalone-ndk.patch', arch.arch)
130136
shprint(perl, 'Configure', *config_args, _env=env)
131137
self.apply_patch('disable-sover.patch', arch.arch)
132138

‎pythonforandroid/recipes/openssl/standalone-ndk.patch

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
--- a/Configurations/15-android.conf 2021-01-02 03:49:06.227255874 +0100
2+
+++ b/Configurations/15-android.conf 2021-01-02 05:57:26.205585631 +0100
3+
@@ -8,7 +8,7 @@
4+
5+
my $android_ndk = {};
6+
my %triplet = (
7+
- arm => "arm-linux-androideabi",
8+
+ arm => "armv7a-linux-androideabi",
9+
arm64 => "aarch64-linux-android",
10+
mips => "mipsel-linux-android",
11+
mips64 => "mips64el-linux-android",
12+
@@ -107,16 +107,19 @@
13+
}
14+
} elsif (-f "$ndk/AndroidVersion.txt") { #"standalone toolchain"
15+
my $cc = $user{CC} // "clang";
16+
+ if ($ENV{NDK_API} =~ m|^android-(\d+)$|) {
17+
+ $api = $1;
18+
+ }
19+
# One can probably argue that both clang and gcc should be
20+
# probed, but support for "standalone toolchain" was added
21+
# *after* announcement that gcc is being phased out, so
22+
# favouring clang is considered adequate. Those who insist
23+
# have option to enforce test for gcc with CC=gcc.
24+
- if (which("$triarch-$cc") !~ m|^$ndk|) {
25+
+ if (which("$triarch$api-$cc") !~ m|^$ndk|) {
26+
die "no NDK $triarch-$cc on \$PATH";
27+
}
28+
- $user{CC} = $cc;
29+
- $user{CROSS_COMPILE} = "$triarch-";
30+
+ $user{CC} = "$triarch$api-$cc";
31+
+ $user{CROSS_COMPILE} = "";
32+
} elsif ($user{CC} eq "clang") {
33+
die "no NDK clang on \$PATH";
34+
} else {

‎pythonforandroid/recipes/pygame/__init__.py

+5-3
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,11 @@ def prebuild_arch(self, arch):
2828
with current_directory(self.get_build_dir(arch.arch)):
2929
setup_template = open(join("buildconfig", "Setup.Android.SDL2.in")).read()
3030
env = self.get_recipe_env(arch)
31-
env['ANDROID_ROOT'] = join(self.ctx.ndk_platform, 'usr')
3231

33-
ndk_lib_dir = join(self.ctx.ndk_platform, 'usr', 'lib')
32+
if self.ctx.ndk_standalone:
33+
pass # FIXME
34+
else:
35+
env['ANDROID_ROOT'] = join(self.ctx.ndk_platform, 'usr')
3436

3537
png = self.get_recipe('png', self.ctx)
3638
png_lib_dir = join(png.get_build_dir(arch.arch), '.libs')
@@ -43,7 +45,7 @@ def prebuild_arch(self, arch):
4345
sdl_includes=(
4446
" -I" + join(self.ctx.bootstrap.build_dir, 'jni', 'SDL', 'include') +
4547
" -L" + join(self.ctx.bootstrap.build_dir, "libs", str(arch)) +
46-
" -L" + png_lib_dir + " -L" + jpeg_lib_dir + " -L" + ndk_lib_dir),
48+
" -L" + png_lib_dir + " -L" + jpeg_lib_dir + " -L" + self.ctx.ndk_lib_dir),
4749
sdl_ttf_includes="-I"+join(self.ctx.bootstrap.build_dir, 'jni', 'SDL2_ttf'),
4850
sdl_image_includes="-I"+join(self.ctx.bootstrap.build_dir, 'jni', 'SDL2_image'),
4951
sdl_mixer_includes="-I"+join(self.ctx.bootstrap.build_dir, 'jni', 'SDL2_mixer'),

‎pythonforandroid/recipes/python3/__init__.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -264,8 +264,8 @@ def add_flags(include_flags, link_dirs, link_libs):
264264
# the build of zlib module, here we search for android's zlib version
265265
# and sets the right flags, so python can be build with android's zlib
266266
info("Activating flags for android's zlib")
267-
zlib_lib_path = join(self.ctx.ndk_platform, 'usr', 'lib')
268-
zlib_includes = join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include')
267+
zlib_lib_path = self.ctx.ndk_lib_dir
268+
zlib_includes = self.ctx.ndk_include_dir
269269
zlib_h = join(zlib_includes, 'zlib.h')
270270
try:
271271
with open(zlib_h) as fileh:

‎pythonforandroid/recipes/vlc/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ def build_arch(self, arch):
5959
env = dict(environ)
6060
env.update({
6161
'ANDROID_ABI': arch.arch,
62-
'ANDROID_NDK': self.ctx.ndk_dir,
62+
'ANDROID_NDK': self.ctx.ndk_dir, # FIXME
6363
'ANDROID_SDK': self.ctx.sdk_dir,
6464
})
6565
info("compiling vlc from sources")

‎pythonforandroid/toolchain.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1052,7 +1052,7 @@ def _build_package(self, args, package_type):
10521052
if not exists("gradlew"):
10531053
raise BuildInterruptingException("gradlew file is missing")
10541054

1055-
env["ANDROID_NDK_HOME"] = self.ctx.ndk_dir
1055+
env['ANDROID_NDK_HOME'] = self.ctx.ndk_dir # FIXME
10561056
env["ANDROID_HOME"] = self.ctx.sdk_dir
10571057

10581058
gradlew = sh.Command('./gradlew')

‎tests/recipes/recipe_ctx.py

+3
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,9 @@ def setUp(self):
4848
f"{self.ctx._ndk_dir}/platforms/"
4949
f"android-{self.ctx.ndk_api}/{self.arch.platform_dir}"
5050
)
51+
self.ctx.ndk_sysroot = f'{self.ctx._ndk_dir}/sysroot'
52+
self.ctx.ndk_lib_dir = f'{self.ctx._ndk_dir}/usr/lib'
53+
self.ctx.ndk_include_dir = f'{self.ctx.ndk_sysroot}/usr/include'
5154
self.recipe = Recipe.get_recipe(self.recipe_name, self.ctx)
5255

5356
def tearDown(self):

0 commit comments

Comments
 (0)