Skip to content

Commit 5475291

Browse files
ToilalToilal
committed
python: use CMake, make or distutils compiler for libjsonnet build
This change takes advantage of distutils compiler feature to compile libjsonnet when neither CMake and make are available, or when both CMake and make build are failing. A fallback strategy to go through this chain of tools (CMake, make, distutils compiler). This also port stdlib/to_c_array.cpp to pure python to generate std.jsonnet.h. Python extension can now be build and installed properly on windows and Python 3.5+ when Visual C++ Build Tools are installed. Windows build on Python 2.7 is still not supported as this version relies on a old version of Visual C++ tools that doesn't support c++11.
1 parent 07c02f3 commit 5475291

File tree

1 file changed

+160
-29
lines changed

1 file changed

+160
-29
lines changed

‎setup.py

Lines changed: 160 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -11,28 +11,42 @@
1111
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
14+
from __future__ import print_function
1415

16+
import errno
1517
import os
16-
from setuptools import setup
18+
import subprocess
19+
from io import open
20+
1721
from setuptools import Extension
18-
from setuptools.command.build_ext import build_ext as BuildExt
19-
from subprocess import Popen
22+
from setuptools import setup
23+
from setuptools.command.build_ext import build_ext
2024

2125
DIR = os.path.abspath(os.path.dirname(__file__))
22-
LIB_OBJECTS = [
23-
'core/desugarer.o',
24-
'core/formatter.o',
25-
'core/libjsonnet.o',
26-
'core/lexer.o',
27-
'core/parser.o',
28-
'core/pass.o',
29-
'core/static_analysis.o',
30-
'core/string_utils.o',
31-
'core/vm.o',
32-
'third_party/md5/md5.o'
33-
]
34-
35-
MODULE_SOURCES = ['python/_jsonnet.c']
26+
27+
28+
def to_c_array(in_file, out_file):
29+
"""
30+
Python port of stdlib/to_c_array.cpp
31+
"""
32+
first_character = True
33+
34+
with open(out_file, 'wb') as out:
35+
with open(in_file, 'rb') as i:
36+
while True:
37+
ba = i.read(1)
38+
if not ba:
39+
break
40+
b = ba[0]
41+
if not isinstance(b, int):
42+
b = ord(b)
43+
if first_character:
44+
first_character = False;
45+
else:
46+
out.write(b',')
47+
out.write(str(b).encode())
48+
out.write(b",0")
49+
3650

3751
def get_version():
3852
"""
@@ -46,19 +60,136 @@ def get_version():
4660
v_code = v_code[1:]
4761
return v_code
4862

49-
class BuildJsonnetExt(BuildExt):
50-
def run(self):
51-
p = Popen(['make'] + LIB_OBJECTS, cwd=DIR)
52-
p.wait()
53-
if p.returncode != 0:
54-
raise Exception('Could not build %s' % (', '.join(LIB_OBJECTS)))
55-
BuildExt.run(self)
63+
64+
def get_cpp_extra_compile_args(compiler):
65+
"""
66+
Get the extra compile arguments for libjsonnet C++ compilation.
67+
"""
68+
if compiler.compiler_type == 'msvc':
69+
return []
70+
else:
71+
return ["-Wextra", "-Woverloaded-virtual", "-pedantic", "-std=c++11"]
72+
73+
74+
def get_c_extra_compile_args(compiler):
75+
"""
76+
Get the extra compile arguments for python jsonnet C compilation.
77+
"""
78+
if compiler.compiler_type == 'msvc':
79+
return []
80+
else:
81+
return ["-Wextra", "-pedantic", "-std=c99"]
82+
83+
84+
def is_cmake_available():
85+
try:
86+
returncode = subprocess.call(["cmake", "--version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
87+
return returncode == 0
88+
except OSError as e:
89+
if e.errno == errno.ENOENT:
90+
return False
91+
raise
92+
93+
94+
def is_make_available():
95+
try:
96+
returncode = subprocess.call(["make", "--version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
97+
return returncode == 0
98+
except OSError as e:
99+
if e.errno == errno.ENOENT:
100+
return False
101+
raise
102+
103+
104+
class custom_build_ext(build_ext):
105+
def build_extension(self, ext):
106+
if ext.name == '_jsonnet':
107+
libjsonnet_built = False
108+
109+
if not libjsonnet_built:
110+
if is_cmake_available():
111+
print("CMake is available.")
112+
print("Building libjsonnet with CMake ...")
113+
subprocess.call(['cmake', '.', '-Bbuild'], cwd=DIR)
114+
returncode = subprocess.call(['cmake', '--build', 'build', '--target', 'libjsonnet_static'],
115+
cwd=DIR)
116+
117+
ext.extra_objects = ['build/core/CMakeFiles/libjsonnet_static.dir/desugarer.cpp.o',
118+
'build/core/CMakeFiles/libjsonnet_static.dir/formatter.cpp.o',
119+
'build/core/CMakeFiles/libjsonnet_static.dir/libjsonnet.cpp.o',
120+
'build/core/CMakeFiles/libjsonnet_static.dir/lexer.cpp.o',
121+
'build/core/CMakeFiles/libjsonnet_static.dir/parser.cpp.o',
122+
'build/core/CMakeFiles/libjsonnet_static.dir/pass.cpp.o',
123+
'build/core/CMakeFiles/libjsonnet_static.dir/static_analysis.cpp.o',
124+
'build/core/CMakeFiles/libjsonnet_static.dir/string_utils.cpp.o',
125+
'build/core/CMakeFiles/libjsonnet_static.dir/vm.cpp.o',
126+
'build/third_party/md5/CMakeFiles/md5.dir/md5.cpp.o']
127+
128+
if returncode == 0:
129+
libjsonnet_built = True
130+
else:
131+
print("libjsonnet build has failed using CMake. Falling back to another build method ...")
132+
else:
133+
print("CMake is not available.")
134+
if not libjsonnet_built:
135+
if is_make_available():
136+
print("make is available.")
137+
print("Building libjsonnet with make ...")
138+
139+
ext.extra_objects = [
140+
'core/desugarer.o',
141+
'core/formatter.o',
142+
'core/libjsonnet.o',
143+
'core/lexer.o',
144+
'core/parser.o',
145+
'core/pass.o',
146+
'core/static_analysis.o',
147+
'core/string_utils.o',
148+
'core/vm.o',
149+
'third_party/md5/md5.o']
150+
151+
returncode = subprocess.call(['make'] + ext.extra_objects, cwd=DIR)
152+
153+
if returncode == 0:
154+
libjsonnet_built = True
155+
else:
156+
print("libjsonnet build has failed using make. Falling back to another build method ...")
157+
else:
158+
print("make is not available")
159+
if not libjsonnet_built:
160+
print("Building libjsonnet with distutils compiler ...")
161+
try:
162+
to_c_array(os.path.join(DIR, 'stdlib/std.jsonnet'), os.path.join(DIR, 'core/std.jsonnet.h'))
163+
164+
ext.extra_objects = self.compiler.compile([
165+
'core/desugarer.cpp',
166+
'core/formatter.cpp',
167+
'core/libjsonnet.cpp',
168+
'core/lexer.cpp',
169+
'core/parser.cpp',
170+
'core/pass.cpp',
171+
'core/static_analysis.cpp',
172+
'core/string_utils.cpp',
173+
'core/vm.cpp',
174+
'third_party/md5/md5.cpp'],
175+
extra_postargs=get_cpp_extra_compile_args(self.compiler),
176+
include_dirs=['include', 'third_party/md5', 'third_party/json'])
177+
178+
libjsonnet_built = True
179+
except Exception as e:
180+
print(e)
181+
print("libjsonnet build has failed using distutils compiler.")
182+
raise Exception("Could not build libjsonnet")
183+
184+
ext.extra_compile_args = get_c_extra_compile_args(self.compiler)
185+
186+
build_ext.build_extension(self, ext)
187+
56188

57189
jsonnet_ext = Extension(
58190
'_jsonnet',
59-
sources=MODULE_SOURCES,
60-
extra_objects=LIB_OBJECTS,
61-
include_dirs = ['include', 'third_party/md5', 'third_party/json'],
191+
sources=['python/_jsonnet.c'],
192+
include_dirs=['include', 'third_party/md5', 'third_party/json'],
62193
language='c++'
63194
)
64195

@@ -69,8 +200,8 @@ def run(self):
69200
author_email='[email protected]',
70201
version=get_version(),
71202
cmdclass={
72-
'build_ext': BuildJsonnetExt,
203+
'build_ext': custom_build_ext,
73204
},
74205
ext_modules=[jsonnet_ext],
75206
test_suite="python._jsonnet_test",
76-
)
207+
)

0 commit comments

Comments
 (0)