From cbb47bfaab9bcb121a161a5fa78194a7331742b1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 27 May 2022 13:56:29 +0100 Subject: [PATCH 01/23] [mypyc] i64 unboxing lib-rt --- mypyc/lib-rt/int_ops.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/mypyc/lib-rt/int_ops.c b/mypyc/lib-rt/int_ops.c index caf0fe0b5391..9683da76beac 100644 --- a/mypyc/lib-rt/int_ops.c +++ b/mypyc/lib-rt/int_ops.c @@ -504,3 +504,19 @@ CPyTagged CPyTagged_Lshift(CPyTagged left, CPyTagged right) { } return CPyTagged_StealFromObject(result); } + +int64_t CPyLong_AsInt64(PyObject *o) { + if (likely(PyLong_Check(o))) { + PyLongObject *lobj = (PyLongObject *)o; + int size = lobj->ob_base.ob_size; + if (size == 1) { + // Fast path + return lobj->ob_digit[0]; + } + } + // Slow path + int overflow; + int64_t result = PyLong_AsLongLongAndOverflow(o, &overflow); + // TODO: Handle errors + return result; +} From a43f0df09c0a93d365cef585b32d9b89b25d467c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 27 May 2022 15:34:18 +0100 Subject: [PATCH 02/23] [mypyc] List indexing with i64 index --- mypyc/lib-rt/list_ops.c | 21 +++++++++++++++++++++ mypyc/primitives/list_ops.py | 13 +++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/list_ops.c b/mypyc/lib-rt/list_ops.c index 885c1a3366f3..337584815e13 100644 --- a/mypyc/lib-rt/list_ops.c +++ b/mypyc/lib-rt/list_ops.c @@ -118,6 +118,27 @@ PyObject *CPyList_GetItemBorrow(PyObject *list, CPyTagged index) { } } +PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index) { + size_t size = PyList_GET_SIZE(list); + if (likely((uint64_t)index < size)) { + PyObject *result = PyList_GET_ITEM(list, index); + Py_INCREF(result); + return result; + } + if (index >= 0) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + index += size; + if (index < 0) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + PyObject *result = PyList_GET_ITEM(list, index); + Py_INCREF(result); + return result; +} + bool CPyList_SetItem(PyObject *list, CPyTagged index, PyObject *value) { if (CPyTagged_CheckShort(index)) { Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); diff --git a/mypyc/primitives/list_ops.py b/mypyc/primitives/list_ops.py index 78955f70f164..18210a3e96aa 100644 --- a/mypyc/primitives/list_ops.py +++ b/mypyc/primitives/list_ops.py @@ -3,7 +3,7 @@ from mypyc.ir.ops import ERR_MAGIC, ERR_NEVER, ERR_FALSE from mypyc.ir.rtypes import ( int_rprimitive, short_int_rprimitive, list_rprimitive, object_rprimitive, c_int_rprimitive, - c_pyssize_t_rprimitive, bit_rprimitive + c_pyssize_t_rprimitive, bit_rprimitive, int64_rprimitive ) from mypyc.primitives.registry import ( load_address_op, function_op, binary_op, method_op, custom_op, ERR_NEG_INT @@ -55,7 +55,7 @@ c_function_name='CPyList_GetItem', error_kind=ERR_MAGIC) -# list[index] version with no int bounds check for when it is known to be short +# list[index] version with no int tag check for when it is known to be short method_op( name='__getitem__', arg_types=[list_rprimitive, short_int_rprimitive], @@ -84,6 +84,15 @@ is_borrowed=True, priority=4) +# Version with native int index +method_op( + name='__getitem__', + arg_types=[list_rprimitive, int64_rprimitive], + return_type=object_rprimitive, + c_function_name='CPyList_GetItemInt64', + error_kind=ERR_MAGIC, + priority=2) + # This is unsafe because it assumes that the index is a non-negative short integer # that is in-bounds for the list. list_get_item_unsafe_op = custom_op( From b642f18224fce43d76d3871f990226bdb0af8295 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 27 May 2022 13:57:28 +0100 Subject: [PATCH 03/23] [mypyc] Overlapping error values lib-rt --- mypyc/lib-rt/int_ops.c | 9 ++++++++- mypyc/lib-rt/mypyc_util.h | 3 +++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/int_ops.c b/mypyc/lib-rt/int_ops.c index 9683da76beac..2d89088be073 100644 --- a/mypyc/lib-rt/int_ops.c +++ b/mypyc/lib-rt/int_ops.c @@ -517,6 +517,13 @@ int64_t CPyLong_AsInt64(PyObject *o) { // Slow path int overflow; int64_t result = PyLong_AsLongLongAndOverflow(o, &overflow); - // TODO: Handle errors + if (result == -1) { + if (PyErr_Occurred()) { + return CPY_LL_INT_ERROR; + } else if (overflow) { + PyErr_SetString(PyExc_OverflowError, "int too large to convert to i64"); + return CPY_LL_INT_ERROR; + } + } return result; } diff --git a/mypyc/lib-rt/mypyc_util.h b/mypyc/lib-rt/mypyc_util.h index 6c4a94f8811c..0fae239cbb9e 100644 --- a/mypyc/lib-rt/mypyc_util.h +++ b/mypyc/lib-rt/mypyc_util.h @@ -53,6 +53,9 @@ typedef PyObject CPyModule; // Tag bit used for long integers #define CPY_INT_TAG 1 +// Error value for fixed-width (low-level) integers +#define CPY_LL_INT_ERROR -113 + typedef void (*CPyVTableItem)(void); static inline CPyTagged CPyTagged_ShortFromInt(int x) { From ad1894554cf09dad8c16a8502a1e905786da8e38 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 20 Sep 2021 11:27:56 +0100 Subject: [PATCH 04/23] Add missing C header declaration --- mypyc/lib-rt/CPy.h | 1 + 1 file changed, 1 insertion(+) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index f482e09cbe79..7b20b4fad071 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -342,6 +342,7 @@ PyObject *CPyList_GetItemUnsafe(PyObject *list, CPyTagged index); PyObject *CPyList_GetItemShort(PyObject *list, CPyTagged index); PyObject *CPyList_GetItemBorrow(PyObject *list, CPyTagged index); PyObject *CPyList_GetItemShortBorrow(PyObject *list, CPyTagged index); +PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index); bool CPyList_SetItem(PyObject *list, CPyTagged index, PyObject *value); bool CPyList_SetItemUnsafe(PyObject *list, CPyTagged index, PyObject *value); PyObject *CPyList_PopLast(PyObject *obj); From d3dd284571dc40afee93edfb30455e01a9f0016f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 27 May 2022 15:35:23 +0100 Subject: [PATCH 05/23] Update i64 division primitive --- mypyc/lib-rt/CPy.h | 1 + mypyc/lib-rt/int_ops.c | 16 ++++++++++++++++ mypyc/primitives/int_ops.py | 10 ++++++++-- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 7b20b4fad071..35f72f242a9a 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -150,6 +150,7 @@ PyObject *CPyLong_FromStrWithBase(PyObject *o, CPyTagged base); PyObject *CPyLong_FromStr(PyObject *o); PyObject *CPyLong_FromFloat(PyObject *o); PyObject *CPyBool_Str(bool b); +int64_t CPyInt64_Divide(int64_t x, int64_t y); static inline int CPyTagged_CheckLong(CPyTagged x) { return x & CPY_INT_TAG; diff --git a/mypyc/lib-rt/int_ops.c b/mypyc/lib-rt/int_ops.c index 2d89088be073..3d15a39b8c3b 100644 --- a/mypyc/lib-rt/int_ops.c +++ b/mypyc/lib-rt/int_ops.c @@ -527,3 +527,19 @@ int64_t CPyLong_AsInt64(PyObject *o) { } return result; } + +int64_t CPyInt64_Divide(int64_t x, int64_t y) { + if (y == 0) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + return CPY_LL_INT_ERROR; + } + if (y == -1 && x == -1L << 63) { + PyErr_SetString(PyExc_OverflowError, "integer division overflow"); + return CPY_LL_INT_ERROR; + } + int64_t d = x / y; + if (((x < 0) != (y < 0)) && d * y != x) { + d--; + } + return d; +} diff --git a/mypyc/primitives/int_ops.py b/mypyc/primitives/int_ops.py index 44703528976c..b0cdc36cb217 100644 --- a/mypyc/primitives/int_ops.py +++ b/mypyc/primitives/int_ops.py @@ -9,10 +9,10 @@ """ from typing import Dict, NamedTuple -from mypyc.ir.ops import ERR_NEVER, ERR_MAGIC, ComparisonOp +from mypyc.ir.ops import ERR_NEVER, ERR_MAGIC, ERR_MAGIC_OVERLAPPING, ComparisonOp from mypyc.ir.rtypes import ( int_rprimitive, bool_rprimitive, float_rprimitive, object_rprimitive, - str_rprimitive, bit_rprimitive, RType + str_rprimitive, bit_rprimitive, int64_rprimitive, RType ) from mypyc.primitives.registry import ( load_address_op, unary_op, CFunctionDescription, function_op, binary_op, custom_op @@ -165,3 +165,9 @@ def int_unary_op(name: str, c_function_name: str) -> CFunctionDescription: '>': IntComparisonOpDescription(ComparisonOp.SGT, int_less_than_, False, True), '>=': IntComparisonOpDescription(ComparisonOp.SGE, int_less_than_, True, False), } + +int64_divide_op = custom_op( + arg_types=[int64_rprimitive, int64_rprimitive], + return_type=int64_rprimitive, + c_function_name='CPyInt64_Divide', + error_kind=ERR_MAGIC_OVERLAPPING) From 3c490b27cd29f310f05e5931893e329a8663cefa Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 27 May 2022 15:36:09 +0100 Subject: [PATCH 06/23] Improvement to // and % primitives --- mypyc/lib-rt/int_ops.c | 18 +++++++++++++++++- mypyc/primitives/int_ops.py | 6 ++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/int_ops.c b/mypyc/lib-rt/int_ops.c index 3d15a39b8c3b..9b1f8e01c9a5 100644 --- a/mypyc/lib-rt/int_ops.c +++ b/mypyc/lib-rt/int_ops.c @@ -533,7 +533,7 @@ int64_t CPyInt64_Divide(int64_t x, int64_t y) { PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); return CPY_LL_INT_ERROR; } - if (y == -1 && x == -1L << 63) { + if (y == -1 && x == -1LL << 63) { PyErr_SetString(PyExc_OverflowError, "integer division overflow"); return CPY_LL_INT_ERROR; } @@ -543,3 +543,19 @@ int64_t CPyInt64_Divide(int64_t x, int64_t y) { } return d; } + +int64_t CPyInt64_Remainder(int64_t x, int64_t y) { + if (y == 0) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + return CPY_LL_INT_ERROR; + } + // Edge case: avoid core dump + if (y == -1 && x == -1LL << 63) { + return 0; + } + int64_t d = x % y; + if (((x < 0) != (y < 0)) && d != 0) { + d += y; + } + return d; +} diff --git a/mypyc/primitives/int_ops.py b/mypyc/primitives/int_ops.py index b0cdc36cb217..8274b8df6756 100644 --- a/mypyc/primitives/int_ops.py +++ b/mypyc/primitives/int_ops.py @@ -171,3 +171,9 @@ def int_unary_op(name: str, c_function_name: str) -> CFunctionDescription: return_type=int64_rprimitive, c_function_name='CPyInt64_Divide', error_kind=ERR_MAGIC_OVERLAPPING) + +int64_mod_op = custom_op( + arg_types=[int64_rprimitive, int64_rprimitive], + return_type=int64_rprimitive, + c_function_name='CPyInt64_Remainder', + error_kind=ERR_MAGIC_OVERLAPPING) From d8018f5c9326c4c49c61ccc5fa0a8a4084394ca0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 11 Oct 2021 09:42:27 +0100 Subject: [PATCH 07/23] Add missing header declaration --- mypyc/lib-rt/CPy.h | 1 + 1 file changed, 1 insertion(+) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 35f72f242a9a..f2820574e2d3 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -151,6 +151,7 @@ PyObject *CPyLong_FromStr(PyObject *o); PyObject *CPyLong_FromFloat(PyObject *o); PyObject *CPyBool_Str(bool b); int64_t CPyInt64_Divide(int64_t x, int64_t y); +int64_t CPyInt64_Remainder(int64_t x, int64_t y); static inline int CPyTagged_CheckLong(CPyTagged x) { return x & CPY_INT_TAG; From b6c1fbd7fe5f3341ffc95dffba9b57c501d7527e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 27 May 2022 15:36:48 +0100 Subject: [PATCH 08/23] i32 // and % primitives --- mypyc/lib-rt/int_ops.c | 36 ++++++++++++++++++++++++++++++++++++ mypyc/primitives/int_ops.py | 14 +++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/int_ops.c b/mypyc/lib-rt/int_ops.c index 9b1f8e01c9a5..411f3e7d9f97 100644 --- a/mypyc/lib-rt/int_ops.c +++ b/mypyc/lib-rt/int_ops.c @@ -538,6 +538,7 @@ int64_t CPyInt64_Divide(int64_t x, int64_t y) { return CPY_LL_INT_ERROR; } int64_t d = x / y; + // Adjust for Python semantics if (((x < 0) != (y < 0)) && d * y != x) { d--; } @@ -554,6 +555,41 @@ int64_t CPyInt64_Remainder(int64_t x, int64_t y) { return 0; } int64_t d = x % y; + // Adjust for Python semantics + if (((x < 0) != (y < 0)) && d != 0) { + d += y; + } + return d; +} + +int32_t CPyInt32_Divide(int32_t x, int32_t y) { + if (y == 0) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + return CPY_LL_INT_ERROR; + } + if (y == -1 && x == -1LL << 31) { + PyErr_SetString(PyExc_OverflowError, "integer division overflow"); + return CPY_LL_INT_ERROR; + } + int32_t d = x / y; + // Adjust for Python semantics + if (((x < 0) != (y < 0)) && d * y != x) { + d--; + } + return d; +} + +int32_t CPyInt32_Remainder(int32_t x, int32_t y) { + if (y == 0) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + return CPY_LL_INT_ERROR; + } + // Edge case: avoid core dump + if (y == -1 && x == -1LL << 31) { + return 0; + } + int32_t d = x % y; + // Adjust for Python semantics if (((x < 0) != (y < 0)) && d != 0) { d += y; } diff --git a/mypyc/primitives/int_ops.py b/mypyc/primitives/int_ops.py index 8274b8df6756..a49c0b22bee5 100644 --- a/mypyc/primitives/int_ops.py +++ b/mypyc/primitives/int_ops.py @@ -12,7 +12,7 @@ from mypyc.ir.ops import ERR_NEVER, ERR_MAGIC, ERR_MAGIC_OVERLAPPING, ComparisonOp from mypyc.ir.rtypes import ( int_rprimitive, bool_rprimitive, float_rprimitive, object_rprimitive, - str_rprimitive, bit_rprimitive, int64_rprimitive, RType + str_rprimitive, bit_rprimitive, int64_rprimitive, int32_rprimitive, RType ) from mypyc.primitives.registry import ( load_address_op, unary_op, CFunctionDescription, function_op, binary_op, custom_op @@ -177,3 +177,15 @@ def int_unary_op(name: str, c_function_name: str) -> CFunctionDescription: return_type=int64_rprimitive, c_function_name='CPyInt64_Remainder', error_kind=ERR_MAGIC_OVERLAPPING) + +int32_divide_op = custom_op( + arg_types=[int32_rprimitive, int32_rprimitive], + return_type=int32_rprimitive, + c_function_name='CPyInt32_Divide', + error_kind=ERR_MAGIC_OVERLAPPING) + +int32_mod_op = custom_op( + arg_types=[int32_rprimitive, int32_rprimitive], + return_type=int32_rprimitive, + c_function_name='CPyInt32_Remainder', + error_kind=ERR_MAGIC_OVERLAPPING) From 3df57c4231e556fecf15da81a4155a922c12549c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 27 May 2022 15:37:25 +0100 Subject: [PATCH 09/23] List set item i64 primitive --- mypyc/lib-rt/list_ops.c | 20 ++++++++++++++++++++ mypyc/primitives/list_ops.py | 9 +++++++++ 2 files changed, 29 insertions(+) diff --git a/mypyc/lib-rt/list_ops.c b/mypyc/lib-rt/list_ops.c index 337584815e13..a67f34133d3b 100644 --- a/mypyc/lib-rt/list_ops.c +++ b/mypyc/lib-rt/list_ops.c @@ -166,6 +166,26 @@ bool CPyList_SetItem(PyObject *list, CPyTagged index, PyObject *value) { } } +bool CPyList_SetItemInt64(PyObject *list, int64_t index, PyObject *value) { + size_t size = PyList_GET_SIZE(list); + if (unlikely((uint64_t)index >= size)) { + if (index > 0) { + PyErr_SetString(PyExc_IndexError, "list assignment index out of range"); + return false; + } + index += size; + if (index < 0) { + PyErr_SetString(PyExc_IndexError, "list assignment index out of range"); + return false; + } + } + // PyList_SET_ITEM doesn't decref the old element, so we do + Py_DECREF(PyList_GET_ITEM(list, index)); + // N.B: Steals reference + PyList_SET_ITEM(list, index, value); + return true; +} + // This function should only be used to fill in brand new lists. bool CPyList_SetItemUnsafe(PyObject *list, CPyTagged index, PyObject *value) { if (CPyTagged_CheckShort(index)) { diff --git a/mypyc/primitives/list_ops.py b/mypyc/primitives/list_ops.py index 18210a3e96aa..d83937aa3fb2 100644 --- a/mypyc/primitives/list_ops.py +++ b/mypyc/primitives/list_ops.py @@ -110,6 +110,15 @@ error_kind=ERR_FALSE, steals=[False, False, True]) +# list[index_i64] = obj +method_op( + name='__setitem__', + arg_types=[list_rprimitive, int64_rprimitive, object_rprimitive], + return_type=bit_rprimitive, + c_function_name='CPyList_SetItemInt64', + error_kind=ERR_FALSE, + steals=[False, False, True]) + # PyList_SET_ITEM does no error checking, # and should only be used to fill in brand new lists. new_list_set_item_op = custom_op( From bd9fd59273809b39f0471a8bde65832eb8b43766 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 11 Oct 2021 14:47:29 +0100 Subject: [PATCH 10/23] Add missing header declarations --- mypyc/lib-rt/CPy.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index f2820574e2d3..c629413fc9d9 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -152,6 +152,8 @@ PyObject *CPyLong_FromFloat(PyObject *o); PyObject *CPyBool_Str(bool b); int64_t CPyInt64_Divide(int64_t x, int64_t y); int64_t CPyInt64_Remainder(int64_t x, int64_t y); +int32_t CPyInt32_Divide(int32_t x, int32_t y); +int32_t CPyInt32_Remainder(int32_t x, int32_t y); static inline int CPyTagged_CheckLong(CPyTagged x) { return x & CPY_INT_TAG; @@ -347,6 +349,7 @@ PyObject *CPyList_GetItemShortBorrow(PyObject *list, CPyTagged index); PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index); bool CPyList_SetItem(PyObject *list, CPyTagged index, PyObject *value); bool CPyList_SetItemUnsafe(PyObject *list, CPyTagged index, PyObject *value); +bool CPyList_SetItemInt64(PyObject *list, int64_t index, PyObject *value); PyObject *CPyList_PopLast(PyObject *obj); PyObject *CPyList_Pop(PyObject *obj, CPyTagged index); CPyTagged CPyList_Count(PyObject *obj, PyObject *value); From 9e8d5d9a69356ff0a84a1faae72f8acf27e670c4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 27 May 2022 15:37:53 +0100 Subject: [PATCH 11/23] int->i64 coercion primitive --- mypyc/primitives/int_ops.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mypyc/primitives/int_ops.py b/mypyc/primitives/int_ops.py index a49c0b22bee5..2950fa1ee2e7 100644 --- a/mypyc/primitives/int_ops.py +++ b/mypyc/primitives/int_ops.py @@ -189,3 +189,9 @@ def int_unary_op(name: str, c_function_name: str) -> CFunctionDescription: return_type=int32_rprimitive, c_function_name='CPyInt32_Remainder', error_kind=ERR_MAGIC_OVERLAPPING) + +int_to_int64_op = custom_op( + arg_types=[object_rprimitive], + return_type=int64_rprimitive, + c_function_name='CPyLong_AsInt64', + error_kind=ERR_MAGIC_OVERLAPPING) From 41bc270a4a9aac110e1c88cff7d8c04e375c43c9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 27 May 2022 14:00:49 +0100 Subject: [PATCH 12/23] [mypyc] Fix CPy.h --- mypyc/lib-rt/CPy.h | 1 + 1 file changed, 1 insertion(+) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index c629413fc9d9..604e42f2852a 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -150,6 +150,7 @@ PyObject *CPyLong_FromStrWithBase(PyObject *o, CPyTagged base); PyObject *CPyLong_FromStr(PyObject *o); PyObject *CPyLong_FromFloat(PyObject *o); PyObject *CPyBool_Str(bool b); +int64_t CPyLong_AsInt64(PyObject *o); int64_t CPyInt64_Divide(int64_t x, int64_t y); int64_t CPyInt64_Remainder(int64_t x, int64_t y); int32_t CPyInt32_Divide(int32_t x, int32_t y); From f306273467eb9f791a518ad1d0e3f5d8f8767402 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 27 May 2022 15:38:35 +0100 Subject: [PATCH 13/23] i64 to int conversion primitive --- mypyc/primitives/int_ops.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mypyc/primitives/int_ops.py b/mypyc/primitives/int_ops.py index 2950fa1ee2e7..14922c44aa7e 100644 --- a/mypyc/primitives/int_ops.py +++ b/mypyc/primitives/int_ops.py @@ -190,8 +190,15 @@ def int_unary_op(name: str, c_function_name: str) -> CFunctionDescription: c_function_name='CPyInt32_Remainder', error_kind=ERR_MAGIC_OVERLAPPING) +# Convert tagged int (as PyObject *) to i64 int_to_int64_op = custom_op( arg_types=[object_rprimitive], return_type=int64_rprimitive, c_function_name='CPyLong_AsInt64', error_kind=ERR_MAGIC_OVERLAPPING) + +int64_to_int_op = custom_op( + arg_types=[int64_rprimitive], + return_type=int_rprimitive, + c_function_name='CPyTagged_FromSsize_t', + error_kind=ERR_MAGIC) From ff03a37d83a5a8cef47fd19f8b2451f823434752 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 27 May 2022 14:03:31 +0100 Subject: [PATCH 14/23] [mypyc] i32 unboxing primitives --- mypyc/lib-rt/CPy.h | 1 + mypyc/lib-rt/int_ops.c | 27 +++++++++++++++++++++++++++ mypyc/primitives/int_ops.py | 7 +++++++ 3 files changed, 35 insertions(+) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 604e42f2852a..f15959e13122 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -153,6 +153,7 @@ PyObject *CPyBool_Str(bool b); int64_t CPyLong_AsInt64(PyObject *o); int64_t CPyInt64_Divide(int64_t x, int64_t y); int64_t CPyInt64_Remainder(int64_t x, int64_t y); +int32_t CPyLong_AsInt32(PyObject *o); int32_t CPyInt32_Divide(int32_t x, int32_t y); int32_t CPyInt32_Remainder(int32_t x, int32_t y); diff --git a/mypyc/lib-rt/int_ops.c b/mypyc/lib-rt/int_ops.c index 411f3e7d9f97..f05c56331451 100644 --- a/mypyc/lib-rt/int_ops.c +++ b/mypyc/lib-rt/int_ops.c @@ -562,6 +562,33 @@ int64_t CPyInt64_Remainder(int64_t x, int64_t y) { return d; } +int32_t CPyLong_AsInt32(PyObject *o) { + if (likely(PyLong_Check(o))) { + PyLongObject *lobj = (PyLongObject *)o; + int size = lobj->ob_base.ob_size; + if (size == 1) { + // Fast path + return lobj->ob_digit[0]; + } + } + // Slow path + int overflow; + long result = PyLong_AsLongAndOverflow(o, &overflow); + if (result > 0x7fffffffLL || result < -0x80000000LL) { + overflow = 1; + result = -1; + } + if (result == -1) { + if (PyErr_Occurred()) { + return CPY_LL_INT_ERROR; + } else if (overflow) { + PyErr_SetString(PyExc_OverflowError, "int too large to convert to i32"); + return CPY_LL_INT_ERROR; + } + } + return result; +} + int32_t CPyInt32_Divide(int32_t x, int32_t y) { if (y == 0) { PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); diff --git a/mypyc/primitives/int_ops.py b/mypyc/primitives/int_ops.py index 14922c44aa7e..f32446a39728 100644 --- a/mypyc/primitives/int_ops.py +++ b/mypyc/primitives/int_ops.py @@ -202,3 +202,10 @@ def int_unary_op(name: str, c_function_name: str) -> CFunctionDescription: return_type=int_rprimitive, c_function_name='CPyTagged_FromSsize_t', error_kind=ERR_MAGIC) + +# Convert tagged int (as PyObject *) to i32 +int_to_int32_op = custom_op( + arg_types=[object_rprimitive], + return_type=int64_rprimitive, + c_function_name='CPyLong_AsInt32', + error_kind=ERR_MAGIC_OVERLAPPING) From d14b33ffb557a37faf634d32232b08a5b43ec51a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 27 May 2022 15:39:07 +0100 Subject: [PATCH 15/23] Update CPy.h --- mypyc/lib-rt/CPy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index f15959e13122..5447342070c2 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -287,7 +287,7 @@ static void CPy_DecRef(PyObject *p) { CPy_DECREF(p); } -CPy_NOINLINE +//CPy_NOINLINE static void CPy_XDecRef(PyObject *p) { CPy_XDECREF(p); } From f6b5be873a40c0bed1e627cdc50ed047df75fe8a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Nov 2021 10:33:58 +0000 Subject: [PATCH 16/23] Return CPy_NOINLINE --- mypyc/lib-rt/CPy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 5447342070c2..f15959e13122 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -287,7 +287,7 @@ static void CPy_DecRef(PyObject *p) { CPy_DECREF(p); } -//CPy_NOINLINE +CPy_NOINLINE static void CPy_XDecRef(PyObject *p) { CPy_XDECREF(p); } From 23d8bf261594dbaf386d577a54698a7cb6cb9998 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 27 May 2022 14:04:41 +0100 Subject: [PATCH 17/23] [mypyc] Coercion int to i32 primitive --- mypyc/lib-rt/CPy.h | 1 + mypyc/lib-rt/int_ops.c | 4 ++++ mypyc/primitives/int_ops.py | 12 +++++++++--- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index f15959e13122..2eb8602ebb88 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -156,6 +156,7 @@ int64_t CPyInt64_Remainder(int64_t x, int64_t y); int32_t CPyLong_AsInt32(PyObject *o); int32_t CPyInt32_Divide(int32_t x, int32_t y); int32_t CPyInt32_Remainder(int32_t x, int32_t y); +void CPyInt32_Overflow(void); static inline int CPyTagged_CheckLong(CPyTagged x) { return x & CPY_INT_TAG; diff --git a/mypyc/lib-rt/int_ops.c b/mypyc/lib-rt/int_ops.c index f05c56331451..7510ba9c9c4f 100644 --- a/mypyc/lib-rt/int_ops.c +++ b/mypyc/lib-rt/int_ops.c @@ -622,3 +622,7 @@ int32_t CPyInt32_Remainder(int32_t x, int32_t y) { } return d; } + +void CPyInt32_Overflow() { + PyErr_SetString(PyExc_OverflowError, "int too large to convert to i32"); +} diff --git a/mypyc/primitives/int_ops.py b/mypyc/primitives/int_ops.py index f32446a39728..b8f6df2c70f7 100644 --- a/mypyc/primitives/int_ops.py +++ b/mypyc/primitives/int_ops.py @@ -9,10 +9,10 @@ """ from typing import Dict, NamedTuple -from mypyc.ir.ops import ERR_NEVER, ERR_MAGIC, ERR_MAGIC_OVERLAPPING, ComparisonOp +from mypyc.ir.ops import ERR_NEVER, ERR_MAGIC, ERR_MAGIC_OVERLAPPING, ERR_ALWAYS, ComparisonOp from mypyc.ir.rtypes import ( int_rprimitive, bool_rprimitive, float_rprimitive, object_rprimitive, - str_rprimitive, bit_rprimitive, int64_rprimitive, int32_rprimitive, RType + str_rprimitive, bit_rprimitive, int64_rprimitive, int32_rprimitive, void_rtype, RType ) from mypyc.primitives.registry import ( load_address_op, unary_op, CFunctionDescription, function_op, binary_op, custom_op @@ -206,6 +206,12 @@ def int_unary_op(name: str, c_function_name: str) -> CFunctionDescription: # Convert tagged int (as PyObject *) to i32 int_to_int32_op = custom_op( arg_types=[object_rprimitive], - return_type=int64_rprimitive, + return_type=int32_rprimitive, c_function_name='CPyLong_AsInt32', error_kind=ERR_MAGIC_OVERLAPPING) + +int32_overflow = custom_op( + arg_types=[], + return_type=void_rtype, + c_function_name='CPyInt32_Overflow', + error_kind=ERR_ALWAYS) From 217ed85f13d1b415e940bf73bc003d0dedbfa409 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 27 May 2022 14:05:55 +0100 Subject: [PATCH 18/23] [mypyc] Primitives fixes --- mypyc/lib-rt/CPy.h | 1 + mypyc/lib-rt/int_ops.c | 9 +++++++++ mypyc/primitives/int_ops.py | 11 +++++++++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 2eb8602ebb88..43234987450c 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -122,6 +122,7 @@ static inline size_t CPy_FindAttrOffset(PyTypeObject *trait, CPyVTableItem *vtab CPyTagged CPyTagged_FromSsize_t(Py_ssize_t value); CPyTagged CPyTagged_FromVoidPtr(void *ptr); +CPyTagged CPyTagged_FromInt64(int64_t value); CPyTagged CPyTagged_FromObject(PyObject *object); CPyTagged CPyTagged_StealFromObject(PyObject *object); CPyTagged CPyTagged_BorrowFromObject(PyObject *object); diff --git a/mypyc/lib-rt/int_ops.c b/mypyc/lib-rt/int_ops.c index 7510ba9c9c4f..8b8ee755a77a 100644 --- a/mypyc/lib-rt/int_ops.c +++ b/mypyc/lib-rt/int_ops.c @@ -35,6 +35,15 @@ CPyTagged CPyTagged_FromVoidPtr(void *ptr) { } } +CPyTagged CPyTagged_FromInt64(int64_t value) { + if (unlikely(CPyTagged_TooBig(value))) { + PyObject *object = PyLong_FromLongLong(value); + return ((CPyTagged)object) | CPY_INT_TAG; + } else { + return value << 1; + } +} + CPyTagged CPyTagged_FromObject(PyObject *object) { int overflow; // The overflow check knows about CPyTagged's width diff --git a/mypyc/primitives/int_ops.py b/mypyc/primitives/int_ops.py index b8f6df2c70f7..ad33de059f02 100644 --- a/mypyc/primitives/int_ops.py +++ b/mypyc/primitives/int_ops.py @@ -12,7 +12,8 @@ from mypyc.ir.ops import ERR_NEVER, ERR_MAGIC, ERR_MAGIC_OVERLAPPING, ERR_ALWAYS, ComparisonOp from mypyc.ir.rtypes import ( int_rprimitive, bool_rprimitive, float_rprimitive, object_rprimitive, - str_rprimitive, bit_rprimitive, int64_rprimitive, int32_rprimitive, void_rtype, RType + str_rprimitive, bit_rprimitive, int64_rprimitive, int32_rprimitive, void_rtype, RType, + c_pyssize_t_rprimitive ) from mypyc.primitives.registry import ( load_address_op, unary_op, CFunctionDescription, function_op, binary_op, custom_op @@ -197,10 +198,16 @@ def int_unary_op(name: str, c_function_name: str) -> CFunctionDescription: c_function_name='CPyLong_AsInt64', error_kind=ERR_MAGIC_OVERLAPPING) +ssize_t_to_int_op = custom_op( + arg_types=[c_pyssize_t_rprimitive], + return_type=int_rprimitive, + c_function_name='CPyTagged_FromSsize_t', + error_kind=ERR_MAGIC) + int64_to_int_op = custom_op( arg_types=[int64_rprimitive], return_type=int_rprimitive, - c_function_name='CPyTagged_FromSsize_t', + c_function_name='CPyTagged_FromInt64', error_kind=ERR_MAGIC) # Convert tagged int (as PyObject *) to i32 From 72807963b267aba4cf690deabd590226721c079d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 1 Jan 2022 12:14:39 +0000 Subject: [PATCH 19/23] Fix coercion on 32-bit arch --- mypyc/lib-rt/CPy.h | 6 ++++++ mypyc/lib-rt/int_ops.c | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 43234987450c..4e3fd8fffb85 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -201,6 +201,12 @@ static inline bool CPyTagged_TooBig(Py_ssize_t value) { && (value >= 0 || value < CPY_TAGGED_MIN); } +static inline bool CPyTagged_TooBigInt64(int64_t value) { + // Micro-optimized for the common case where it fits. + return (uint64_t)value > CPY_TAGGED_MAX + && (value >= 0 || value < CPY_TAGGED_MIN); +} + static inline bool CPyTagged_IsAddOverflow(CPyTagged sum, CPyTagged left, CPyTagged right) { // This check was copied from some of my old code I believe that it works :-) return (Py_ssize_t)(sum ^ left) < 0 && (Py_ssize_t)(sum ^ right) < 0; diff --git a/mypyc/lib-rt/int_ops.c b/mypyc/lib-rt/int_ops.c index 8b8ee755a77a..b715e8ab85ed 100644 --- a/mypyc/lib-rt/int_ops.c +++ b/mypyc/lib-rt/int_ops.c @@ -36,7 +36,7 @@ CPyTagged CPyTagged_FromVoidPtr(void *ptr) { } CPyTagged CPyTagged_FromInt64(int64_t value) { - if (unlikely(CPyTagged_TooBig(value))) { + if (unlikely(CPyTagged_TooBigInt64(value))) { PyObject *object = PyLong_FromLongLong(value); return ((CPyTagged)object) | CPY_INT_TAG; } else { From 139f11cebbeb787e2553012b0a6222c72b812fee Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 27 May 2022 15:39:40 +0100 Subject: [PATCH 20/23] i64 list get item with borrowing --- mypyc/lib-rt/CPy.h | 1 + mypyc/lib-rt/list_ops.c | 17 +++++++++++++++++ mypyc/primitives/list_ops.py | 10 ++++++++++ 3 files changed, 28 insertions(+) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 4e3fd8fffb85..ca8bc31140af 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -357,6 +357,7 @@ PyObject *CPyList_GetItemShort(PyObject *list, CPyTagged index); PyObject *CPyList_GetItemBorrow(PyObject *list, CPyTagged index); PyObject *CPyList_GetItemShortBorrow(PyObject *list, CPyTagged index); PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index); +PyObject *CPyList_GetItemInt64Borrow(PyObject *list, int64_t index); bool CPyList_SetItem(PyObject *list, CPyTagged index, PyObject *value); bool CPyList_SetItemUnsafe(PyObject *list, CPyTagged index, PyObject *value); bool CPyList_SetItemInt64(PyObject *list, int64_t index, PyObject *value); diff --git a/mypyc/lib-rt/list_ops.c b/mypyc/lib-rt/list_ops.c index a67f34133d3b..cb72662e22ee 100644 --- a/mypyc/lib-rt/list_ops.c +++ b/mypyc/lib-rt/list_ops.c @@ -139,6 +139,23 @@ PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index) { return result; } +PyObject *CPyList_GetItemInt64Borrow(PyObject *list, int64_t index) { + size_t size = PyList_GET_SIZE(list); + if (likely((uint64_t)index < size)) { + return PyList_GET_ITEM(list, index); + } + if (index >= 0) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + index += size; + if (index < 0) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + return PyList_GET_ITEM(list, index); +} + bool CPyList_SetItem(PyObject *list, CPyTagged index, PyObject *value) { if (CPyTagged_CheckShort(index)) { Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); diff --git a/mypyc/primitives/list_ops.py b/mypyc/primitives/list_ops.py index d83937aa3fb2..a8bcaaadddc0 100644 --- a/mypyc/primitives/list_ops.py +++ b/mypyc/primitives/list_ops.py @@ -93,6 +93,16 @@ error_kind=ERR_MAGIC, priority=2) +# Version with native int index +method_op( + name='__getitem__', + arg_types=[list_rprimitive, int64_rprimitive], + return_type=object_rprimitive, + c_function_name='CPyList_GetItemInt64Borrow', + is_borrowed=True, + error_kind=ERR_MAGIC, + priority=4) + # This is unsafe because it assumes that the index is a non-negative short integer # that is in-bounds for the list. list_get_item_unsafe_op = custom_op( From cdd09f8b884eadb9f362b6fefe023a48f5379b54 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 27 May 2022 14:12:18 +0100 Subject: [PATCH 21/23] [mypyc] Primitives change --- mypyc/primitives/list_ops.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypyc/primitives/list_ops.py b/mypyc/primitives/list_ops.py index a8bcaaadddc0..dd133af678f8 100644 --- a/mypyc/primitives/list_ops.py +++ b/mypyc/primitives/list_ops.py @@ -127,7 +127,8 @@ return_type=bit_rprimitive, c_function_name='CPyList_SetItemInt64', error_kind=ERR_FALSE, - steals=[False, False, True]) + steals=[False, False, True], + priority=2) # PyList_SET_ITEM does no error checking, # and should only be used to fill in brand new lists. From 32dd55fe8984154ae1bb9ae1ef9ed30f801fe6ca Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 22 May 2022 11:44:06 +0100 Subject: [PATCH 22/23] Faster unboxing to i64/i32 --- mypyc/lib-rt/int_ops.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mypyc/lib-rt/int_ops.c b/mypyc/lib-rt/int_ops.c index b715e8ab85ed..42e6908384f6 100644 --- a/mypyc/lib-rt/int_ops.c +++ b/mypyc/lib-rt/int_ops.c @@ -517,10 +517,12 @@ CPyTagged CPyTagged_Lshift(CPyTagged left, CPyTagged right) { int64_t CPyLong_AsInt64(PyObject *o) { if (likely(PyLong_Check(o))) { PyLongObject *lobj = (PyLongObject *)o; - int size = lobj->ob_base.ob_size; - if (size == 1) { + Py_ssize_t size = Py_SIZE(lobj); + if (likely(size == 1)) { // Fast path return lobj->ob_digit[0]; + } else if (likely(size == 0)) { + return 0; } } // Slow path @@ -574,10 +576,12 @@ int64_t CPyInt64_Remainder(int64_t x, int64_t y) { int32_t CPyLong_AsInt32(PyObject *o) { if (likely(PyLong_Check(o))) { PyLongObject *lobj = (PyLongObject *)o; - int size = lobj->ob_base.ob_size; - if (size == 1) { + Py_ssize_t size = lobj->ob_base.ob_size; + if (likely(size == 1)) { // Fast path return lobj->ob_digit[0]; + } else if (likely(size == 0)) { + return 0; } } // Slow path From 09581eea7214c130919f015680b2d0209c801238 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 27 May 2022 10:36:34 +0100 Subject: [PATCH 23/23] Fix ambiguous priorities of list get item primitives --- mypyc/primitives/list_ops.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/primitives/list_ops.py b/mypyc/primitives/list_ops.py index dd133af678f8..2bba4207cd27 100644 --- a/mypyc/primitives/list_ops.py +++ b/mypyc/primitives/list_ops.py @@ -91,7 +91,7 @@ return_type=object_rprimitive, c_function_name='CPyList_GetItemInt64', error_kind=ERR_MAGIC, - priority=2) + priority=5) # Version with native int index method_op( @@ -101,7 +101,7 @@ c_function_name='CPyList_GetItemInt64Borrow', is_borrowed=True, error_kind=ERR_MAGIC, - priority=4) + priority=6) # This is unsafe because it assumes that the index is a non-negative short integer # that is in-bounds for the list.