Update python to match cpp version

Author: sparkprime

bazel/repositories.bzl

@@ -4,8 +4,8 @@ load(
 )
 
 # NB: update_cpp_jsonnet.sh looks for these.
-CPP_JSONNET_SHA256 = "21ebdb2d9e3ac83f5ee80a94ef37112b412440407e2f3db8e8147544a64b8ae1"
-CPP_JSONNET_GITHASH = "ca2d672ffe4c243570671ee0cd62d887f123372e"
+CPP_JSONNET_SHA256 = "2f75785fcb66ac9d0fc05cbb8861263ec35a815bba574f0ec98462a74f6eed3d"
+CPP_JSONNET_GITHASH = "739d3ecafa2c6100ec97559751656a5e1f8488d4"
 
 def jsonnet_go_repositories():
     http_archive(

cpp-jsonnet

@@ -147,7 +147,6 @@ static struct JsonnetJsonValue *cpython_native_callback(
     void *ctx_, const struct JsonnetJsonValue * const *argv, int *succ)
 {
     const struct NativeCtx *ctx = ctx_;
-    int i;
 
     PyEval_RestoreThread(*ctx->py_thread);
 
@@ -156,7 +155,7 @@ static struct JsonnetJsonValue *cpython_native_callback(
 
     // Populate python function args.
     arglist = PyTuple_New(ctx->argc);
-    for (i = 0; i < ctx->argc; ++i) {
+    for (size_t i = 0; i < ctx->argc; ++i) {
         double d;
         const char *param_str = jsonnet_json_extract_string(ctx->vm, argv[i]);
         int param_null = jsonnet_json_extract_null(ctx->vm, argv[i]);
@@ -250,22 +249,21 @@ static int cpython_import_callback(void *ctx_, const char *base, const char *rel
 #if PY_MAJOR_VERSION >= 3
         if (!PyUnicode_Check(file_name) || !PyBytes_Check(file_content)) {
 #else
-        if (!PyString_Check(file_name) || !PyString_Check(file_content)) {
+        if (!PyString_Check(file_name) || !PyBytes_Check(file_content)) {
 #endif
-            *buf = jsonnet_str_nonull(ctx->vm, "import_callback did not return (string, bytes)", buflen);
+            *buf = jsonnet_str_nonull(ctx->vm, "import_callback did not return (string, bytes). Since 0.19.0 imports should be returned as bytes instead of as a string.  You may want to call .encode() on your string.", buflen);
             success = 0;
         } else {
-            const char *content_buf;
-            const ssize_t content_len;
+            char *content_buf;
+            ssize_t content_len;
 #if PY_MAJOR_VERSION >= 3
             const char *found_here_cstr = PyUnicode_AsUTF8(file_name);
-            PyBytes_AsStringAndSize(file_content, &content_buf, &content_len);
 #else
             const char *found_here_cstr = PyString_AsString(file_name);
-            PyString_AsStringAndSize(file_content, &content_buf, &content_len);
 #endif
+            PyBytes_AsStringAndSize(file_content, &content_buf, &content_len);
             *found_here = jsonnet_str(ctx->vm, found_here_cstr);
-            *buflen = content_len - 1; // Python always adds a trailing null
+            *buflen = content_len;
             *buf = jsonnet_realloc(ctx->vm, NULL, *buflen);
             memcpy(*buf, content_buf, *buflen);
             success = 1;
@@ -471,7 +469,8 @@ static int handle_native_callbacks(struct JsonnetVm *vm, PyObject *native_callba
 static PyObject* evaluate_file(PyObject* self, PyObject* args, PyObject *keywds)
 {
     const char *filename;
-    char *out, *jpath_str;
+    char *out;
+    const char *jpath_str;
     unsigned max_stack = 500, gc_min_objects = 1000, max_trace = 20;
     double gc_growth_trigger = 2;
     int error;
@@ -560,7 +559,8 @@ static PyObject* evaluate_file(PyObject* self, PyObject* args, PyObject *keywds)
 static PyObject* evaluate_snippet(PyObject* self, PyObject* args, PyObject *keywds)
 {
     const char *filename, *src;
-    char *out, *jpath_str;
+    char *out;
+    const char *jpath_str;
     unsigned max_stack = 500, gc_min_objects = 1000, max_trace = 20;
     double gc_growth_trigger = 2;
     int error;
@@ -654,7 +654,7 @@ static PyMethodDef module_methods[] = {
 };
 
 #if PY_MAJOR_VERSION >= 3
-static struct PyModuleDef _gojsonnet =
+static struct PyModuleDef _module =
 {
     PyModuleDef_HEAD_INIT,
     "_gojsonnet",
@@ -665,11 +665,20 @@ static struct PyModuleDef _gojsonnet =
 
 PyMODINIT_FUNC PyInit__gojsonnet(void)
 {
-    return PyModule_Create(&_gojsonnet);
+    PyObject *module =  PyModule_Create(&_module);
+    PyObject *version_str = PyUnicode_FromString(LIB_JSONNET_VERSION);
+    if (PyModule_AddObject(module, "version", PyUnicode_FromString(LIB_JSONNET_VERSION)) < 0) {
+      Py_XDECREF(version_str);
+    }
+    return module;
 }
 #else
 PyMODINIT_FUNC init_gojsonnet(void)
 {
-    Py_InitModule3("_gojsonnet", module_methods, "A Python interface to Jsonnet.");
+    PyObject *module = Py_InitModule3("_gojsonnet", module_methods, "A Python interface to Jsonnet.");
+    PyObject *version_str = PyUnicode_FromString(LIB_JSONNET_VERSION);
+    if (PyModule_AddObject(module, "version", PyString_FromString(LIB_JSONNET_VERSION)) < 0) {
+      Py_XDECREF(version_str);
+    }
 }
 #endif

python/_jsonnet.c

@@ -13,10 +13,12 @@
 # limitations under the License.
 
 import os
+import sys
 import unittest
 
 import _gojsonnet
 
+
 # Returns (full_path, contents) if the file was successfully retrieved,
 # (full_path, None) if file not found, or throws an exception when the path
 # is invalid or an IO error occured.
@@ -39,14 +41,16 @@ def try_path_cached(cache, dir, rel):
                 cache[full_path] = f.read().encode()
     return full_path, cache[full_path]
 
-
-def import_callback(dir, rel):
+def import_callback_encode(dir, rel):
     cache = {}
     full_path, content = try_path_cached(cache, dir, rel)
     if content:
         return full_path, content
     raise RuntimeError('File not found')
 
+def import_callback_empty_file_encode(dir, rel):
+    return dir, b''
+
 
 # Test native extensions
 def concat(a, b):
@@ -81,39 +85,96 @@ def setUp(self):
         with open(self.input_filename, "r") as infile:
             self.input_snippet = infile.read()
 
-    def test_evaluate_file(self):
+    def test_version(self):
+        self.assertEqual(type(_gojsonnet.version), str)
+
+    def test_evaluate_file_encode(self):
         json_str = _gojsonnet.evaluate_file(
             self.input_filename,
-            import_callback=import_callback,
+            import_callback=import_callback_encode,
             native_callbacks=native_callbacks,
         )
         self.assertEqual(json_str, "true\n")
 
-    def test_evaluate_snippet(self):
+    def test_evaluate_snippet_encode(self):
         json_str = _gojsonnet.evaluate_snippet(
             self.test_filename,
             self.input_snippet,
-            import_callback=import_callback,
+            import_callback=import_callback_encode,
             native_callbacks=native_callbacks,
         )
         self.assertEqual(json_str, "true\n")
 
-    def test_import(self):
+    def test_evaluate_snippet_encode(self):
+        json_str = _gojsonnet.evaluate_snippet(
+            self.test_filename,
+            self.input_snippet,
+            import_callback=import_callback_encode,
+            native_callbacks=native_callbacks,
+        )
+        self.assertEqual(json_str, "true\n")
+
+    def test_import_encode(self):
         json_str = _gojsonnet.evaluate_snippet(
             self.test_filename,
             "import 'trivial.jsonnet'",
-            import_callback=import_callback,
+            import_callback=import_callback_encode,
             native_callbacks=native_callbacks,
         )
         self.assertEqual(json_str, "42\n")
 
+    def test_import_no_eol_encode(self):
+        json_str = _gojsonnet.evaluate_snippet(
+            self.test_filename,
+            "import 'trivial_no_eol.jsonnet'",
+            import_callback=import_callback_encode,
+            native_callbacks=native_callbacks,
+        )
+        self.assertEqual(json_str, "42\n")
+
+    def test_import_binary_encode(self):
+        json_str = _gojsonnet.evaluate_snippet(
+            self.test_filename,
+            "importbin 'binary123.bin'",
+            import_callback=import_callback_encode,
+            native_callbacks=native_callbacks,
+        )
+        self.assertEqual(json_str, "[\n   1,\n   2,\n   3\n]\n")
+
+    def test_import_binary_sentinel_encode(self):
+        json_str = _gojsonnet.evaluate_snippet(
+            self.test_filename,
+            "importbin 'binary1230123.bin'",
+            import_callback=import_callback_encode,
+            native_callbacks=native_callbacks,
+        )
+        self.assertEqual(json_str, "[\n   1,\n   2,\n   3,\n   0,\n   1,\n   2,\n   3\n]\n")
+
+    def test_import_str_empty_file_encode(self):
+        json_str = _gojsonnet.evaluate_snippet(
+            self.test_filename,
+            "importstr 'binary123.bin'",
+            import_callback=import_callback_empty_file_encode,
+            native_callbacks=native_callbacks,
+        )
+        self.assertEqual(json_str, "\"\"\n")
+
+    def test_import_binary_empty_file_encode(self):
+        json_str = _gojsonnet.evaluate_snippet(
+            self.test_filename,
+            "importbin 'binary123.bin'",
+            import_callback=import_callback_empty_file_encode,
+            native_callbacks=native_callbacks,
+        )
+        self.assertEqual(json_str, "[ ]\n")
+
     def test_double_import(self):
         json_str = _gojsonnet.evaluate_snippet(
             self.test_filename,
             "local x = import 'trivial.jsonnet';\n" +
             "local y = import 'trivial.jsonnet';\n" +
             "x + y",
-            import_callback=import_callback,
+            import_callback=import_callback_encode,
             native_callbacks=native_callbacks,
         )
         self.assertEqual(json_str, "84\n")

python/_jsonnet_test.py

@@ -0,0 +1 @@
+


python/testdata/binary123.bin

@@ -0,0 +1,2 @@
+// used for testing imports
+42