From f19174d20fc67810182dc492eafd5b621b69277e Mon Sep 17 00:00:00 2001 From: Guanlan Dai Date: Tue, 25 Jun 2013 17:45:36 -0700 Subject: [PATCH 01/10] add preallocate table support --- lua_cmsgpack.c | 115 +++++++++++++++++++++++++++++++++++++++++++++---- test.lua | 1 + 2 files changed, 108 insertions(+), 8 deletions(-) diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index 9061213..08ebdf8 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -367,26 +367,24 @@ static void mp_encode_lua_table_as_map(lua_State *L, mp_buf *buf, int level) { * of keys from numerical keys from 1 up to N, with N being the total number * of elements, without any hole in the middle. */ static int table_is_an_array(lua_State *L) { - long count = 0, max = 0, idx = 0; + long idx = 0; lua_Number n; lua_pushnil(L); while(lua_next(L,-2)) { + idx++; /* Stack: ... key value */ lua_pop(L,1); /* Stack: ... key */ if (!lua_isnumber(L,-1)) goto not_array; n = lua_tonumber(L,-1); - idx = n; if (idx != n || idx < 1) goto not_array; - count++; - max = idx; } /* We have the total number of elements in "count". Also we have * the max index encountered in "idx". We can't reach this code * if there are indexes <= 0. If you also note that there can not be * repeated keys into a table, you have that if idx==count you are sure * that there are all the keys form 1 to count (both included). */ - return idx == count; + return 1; not_array: lua_pop(L,1); @@ -437,12 +435,108 @@ static int mp_pack(lua_State *L) { /* --------------------------------- Decoding --------------------------------- */ +unsigned int next_power_of_two(int n) +{ + n--; + n |= n >> 1; + n |= n >> 2; + n |= n >> 4; + n |= n >> 8; + n |= n >> 16; + n++; + return n; +} + +int get_narray(mp_cur *c, size_t len) { + const unsigned char *curr = c->p; + int narray = 0; + int i; + int idx = 0; + int value; + for (i = 0; i < len * 2; i++) { + switch (curr[0]) { + case 0xcc: /* uint 8 */ + case 0xd0: /* int 8 */ + if (i % 2 == 0) { + idx++; + if (idx == curr[1] || curr[1] <= next_power_of_two(idx)) + narray++; + else + return narray; + } + curr += 2; + break; + case 0xcd: /* uint 16 */ + case 0xd1: /* int 16 */ + if (i % 2 == 0) { + idx++; + value = (curr[1] << 8) | curr[2]; + if (idx == value || value <= next_power_of_two(idx)) + narray++; + else + return narray; + } + curr += 3; + break; + case 0xce: /* uint 32 */ + case 0xd2: /* int 32 */ + if (i % 2 == 0) { + idx++; + value = ((uint32_t)curr[1] << 24) | + ((uint32_t)curr[2] << 16) | + ((uint32_t)curr[3] << 8) | + (uint32_t)curr[4]; + if (idx == value || value <= next_power_of_two(idx)) + narray++; + else + return narray; + } + curr += 5; + break; + case 0xcf: /* uint 64 */ + case 0xd3: /* int 64 */ + if (i % 2 == 0) { + idx++; + value = ((uint64_t)curr[1] << 56) | + ((uint64_t)curr[2] << 48) | + ((uint64_t)curr[3] << 40) | + ((uint64_t)curr[4] << 32) | + ((uint64_t)curr[5] << 24) | + ((uint64_t)curr[6] << 16) | + ((uint64_t)curr[7] << 8) | + (uint64_t)curr[8]; + if (idx == value || value <= next_power_of_two(idx)) + narray++; + else + return narray; + } + curr += 9; + break; + default: /* types that can't be idenitified by first byte value. */ + if ((curr[0] & 0x80) == 0) { /* positive fixnum */ + curr += 1; + if (i % 2 == 0) { + idx++; + if (idx == curr[0] || curr[0] <= next_power_of_two(idx)) { + narray++; + } else { + return narray; + } + } + } else { /* negative fixnum */ + return narray; + } + } + } + return narray; +} + void mp_decode_to_lua_type(lua_State *L, mp_cur *c); void mp_decode_to_lua_array(lua_State *L, mp_cur *c, size_t len) { int index = 1; - - lua_newtable(L); + /*lua_newtable(L);*/ + lua_createtable(L, len, 0); while(len--) { lua_pushnumber(L,index++); mp_decode_to_lua_type(L,c); @@ -452,7 +546,12 @@ void mp_decode_to_lua_array(lua_State *L, mp_cur *c, size_t len) { } void mp_decode_to_lua_hash(lua_State *L, mp_cur *c, size_t len) { - lua_newtable(L); + int narray; + int nhash; + narray = get_narray(c, len); + nhash = len - narray; + lua_createtable(L, narray, nhash); + /*lua_newtable(L);*/ while(len--) { mp_decode_to_lua_type(L,c); /* key */ if (c->err) return; diff --git a/test.lua b/test.lua index 8bec0f5..557678b 100644 --- a/test.lua +++ b/test.lua @@ -1,3 +1,4 @@ +require "cmsgpack" -- lua_cmsgpack.c lib tests -- Copyright(C) 2012 Salvatore Sanfilippo, All Rights Reserved. -- See the copyright notice at the end of lua_cmsgpack.c for more information. From cb5b233415379ca767299c64f9759e0c3d1275e6 Mon Sep 17 00:00:00 2001 From: Guanlan Dai Date: Tue, 25 Jun 2013 17:45:59 -0700 Subject: [PATCH 02/10] add Makefile --- Makefile | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1c8c2c8 --- /dev/null +++ b/Makefile @@ -0,0 +1,52 @@ +##### Available defines for CJSON_CFLAGS ##### +## +## USE_INTERNAL_ISINF: Workaround for Solaris platforms missing isinf(). +## DISABLE_INVALID_NUMBERS: Permanently disable invalid JSON numbers: +## NaN, Infinity, hex. +## +## Optional built-in number conversion uses the following defines: +## USE_INTERNAL_FPCONV: Use builtin strtod/dtoa for numeric conversions. +## IEEE_BIG_ENDIAN: Required on big endian architectures. +## MULTIPLE_THREADS: Must be set when Lua CJSON may be used in a +## multi-threaded application. Requries _pthreads_. + +##### Build defaults ##### +LUA_VERSION = 5.1 +TARGET = cmsgpack.so +PREFIX = /usr/local +#CFLAGS = -g -Wall -pedantic -fno-inline +CFLAGS = -O3 -Wall -pedantic -DNDEBUG +CJSON_CFLAGS = -fpic +CJSON_LDFLAGS = -shared +LUA_INCLUDE_DIR = $(PREFIX)/include +LUA_CMODULE_DIR = $(PREFIX)/lib/lua/$(LUA_VERSION) +LUA_MODULE_DIR = $(PREFIX)/share/lua/$(LUA_VERSION) +LUA_BIN_DIR = $(PREFIX)/bin + + +##### End customisable sections ##### + +DATAPERM = 644 +EXECPERM = 755 + +ASCIIDOC = asciidoc + +BUILD_CFLAGS = -I$(LUA_INCLUDE_DIR) $(CJSON_CFLAGS) +OBJS = lua_cmsgpack.o +.PHONY: all clean install + +%.o: %.c + $(CC) -c $(CFLAGS) $(BUILD_CFLAGS) -o $@ $< + +all: $(TARGET) + +$(TARGET): $(OBJS) + $(CC) $(CJSON_LDFLAGS) -o $@ $(OBJS) + +install: $(TARGET) + mkdir -p $(DESTDIR)/$(LUA_CMODULE_DIR) + cp $(TARGET) $(DESTDIR)/$(LUA_CMODULE_DIR) + chmod $(EXECPERM) $(DESTDIR)/$(LUA_CMODULE_DIR)/$(TARGET) + +clean: + rm -f *.o $(TARGET) From 0c2bea9b8c40dd49ca9cf22d6703e04d8c4ee8b3 Mon Sep 17 00:00:00 2001 From: Guanlan Dai Date: Wed, 26 Jun 2013 16:31:58 -0700 Subject: [PATCH 03/10] fix a minor bug --- Makefile | 16 ++++++++-------- lua_cmsgpack.c | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 1c8c2c8..6713344 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -##### Available defines for CJSON_CFLAGS ##### +##### Available defines for CMSGPACK_CFLAGS ##### ## ## USE_INTERNAL_ISINF: Workaround for Solaris platforms missing isinf(). ## DISABLE_INVALID_NUMBERS: Permanently disable invalid JSON numbers: @@ -7,17 +7,17 @@ ## Optional built-in number conversion uses the following defines: ## USE_INTERNAL_FPCONV: Use builtin strtod/dtoa for numeric conversions. ## IEEE_BIG_ENDIAN: Required on big endian architectures. -## MULTIPLE_THREADS: Must be set when Lua CJSON may be used in a +## MULTIPLE_THREADS: Must be set when Lua CMSGPACK may be used in a ## multi-threaded application. Requries _pthreads_. ##### Build defaults ##### LUA_VERSION = 5.1 TARGET = cmsgpack.so PREFIX = /usr/local -#CFLAGS = -g -Wall -pedantic -fno-inline -CFLAGS = -O3 -Wall -pedantic -DNDEBUG -CJSON_CFLAGS = -fpic -CJSON_LDFLAGS = -shared +CFLAGS = -g -Wall -pedantic -fno-inline +#CFLAGS = -O3 -Wall -pedantic -DNDEBUG +CMSGPACK_CFLAGS = -fpic +CMSGPACK_LDFLAGS = -shared LUA_INCLUDE_DIR = $(PREFIX)/include LUA_CMODULE_DIR = $(PREFIX)/lib/lua/$(LUA_VERSION) LUA_MODULE_DIR = $(PREFIX)/share/lua/$(LUA_VERSION) @@ -31,7 +31,7 @@ EXECPERM = 755 ASCIIDOC = asciidoc -BUILD_CFLAGS = -I$(LUA_INCLUDE_DIR) $(CJSON_CFLAGS) +BUILD_CFLAGS = -I$(LUA_INCLUDE_DIR) $(CMSGPACK_CFLAGS) OBJS = lua_cmsgpack.o .PHONY: all clean install @@ -41,7 +41,7 @@ OBJS = lua_cmsgpack.o all: $(TARGET) $(TARGET): $(OBJS) - $(CC) $(CJSON_LDFLAGS) -o $@ $(OBJS) + $(CC) $(CMSGPACK_LDFLAGS) -o $@ $(OBJS) install: $(TARGET) mkdir -p $(DESTDIR)/$(LUA_CMODULE_DIR) diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index 08ebdf8..6f6bda6 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -514,7 +514,6 @@ int get_narray(mp_cur *c, size_t len) { break; default: /* types that can't be idenitified by first byte value. */ if ((curr[0] & 0x80) == 0) { /* positive fixnum */ - curr += 1; if (i % 2 == 0) { idx++; if (idx == curr[0] || curr[0] <= next_power_of_two(idx)) { @@ -523,7 +522,8 @@ int get_narray(mp_cur *c, size_t len) { return narray; } } - } else { /* negative fixnum */ + curr += 1; + } else { /* other */ return narray; } } From c1d5841e95a2e8b2111f520354f922c615842b0b Mon Sep 17 00:00:00 2001 From: Guanlan Dai Date: Mon, 1 Jul 2013 17:55:51 -0700 Subject: [PATCH 04/10] rename get_narray to calc_arrary_num --- lua_cmsgpack.c | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index 6f6bda6..7d87512 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -447,8 +447,9 @@ unsigned int next_power_of_two(int n) return n; } -int get_narray(mp_cur *c, size_t len) { +int calc_array_num(mp_cur *c, size_t len) { const unsigned char *curr = c->p; + size_t left = c->left; int narray = 0; int i; int idx = 0; @@ -459,17 +460,25 @@ int get_narray(mp_cur *c, size_t len) { case 0xd0: /* int 8 */ if (i % 2 == 0) { idx++; + if (left < 2) { + return narray; + } + if (idx == curr[1] || curr[1] <= next_power_of_two(idx)) narray++; else return narray; } curr += 2; + left -=2; break; case 0xcd: /* uint 16 */ case 0xd1: /* int 16 */ if (i % 2 == 0) { idx++; + if (left < 3) { + return narray; + } value = (curr[1] << 8) | curr[2]; if (idx == value || value <= next_power_of_two(idx)) narray++; @@ -477,11 +486,15 @@ int get_narray(mp_cur *c, size_t len) { return narray; } curr += 3; + left -= 3; break; case 0xce: /* uint 32 */ case 0xd2: /* int 32 */ if (i % 2 == 0) { idx++; + if (left < 5) { + return narray; + } value = ((uint32_t)curr[1] << 24) | ((uint32_t)curr[2] << 16) | ((uint32_t)curr[3] << 8) | @@ -492,11 +505,16 @@ int get_narray(mp_cur *c, size_t len) { return narray; } curr += 5; + left -= 5; break; case 0xcf: /* uint 64 */ case 0xd3: /* int 64 */ if (i % 2 == 0) { idx++; + if (left < 9) { + return narray; + } + value = ((uint64_t)curr[1] << 56) | ((uint64_t)curr[2] << 48) | ((uint64_t)curr[3] << 40) | @@ -511,6 +529,7 @@ int get_narray(mp_cur *c, size_t len) { return narray; } curr += 9; + left -= 9; break; default: /* types that can't be idenitified by first byte value. */ if ((curr[0] & 0x80) == 0) { /* positive fixnum */ @@ -523,6 +542,7 @@ int get_narray(mp_cur *c, size_t len) { } } curr += 1; + left -= 1; } else { /* other */ return narray; } @@ -535,8 +555,8 @@ void mp_decode_to_lua_type(lua_State *L, mp_cur *c); void mp_decode_to_lua_array(lua_State *L, mp_cur *c, size_t len) { int index = 1; - /*lua_newtable(L);*/ - lua_createtable(L, len, 0); + /*lua_newtable(L);*/ + lua_createtable(L, len, 0); while(len--) { lua_pushnumber(L,index++); mp_decode_to_lua_type(L,c); @@ -548,7 +568,7 @@ void mp_decode_to_lua_array(lua_State *L, mp_cur *c, size_t len) { void mp_decode_to_lua_hash(lua_State *L, mp_cur *c, size_t len) { int narray; int nhash; - narray = get_narray(c, len); + narray = calc_array_num(c, len); nhash = len - narray; lua_createtable(L, narray, nhash); /*lua_newtable(L);*/ From f8f3ad095661df6971fde56383720734781da2ea Mon Sep 17 00:00:00 2001 From: guanlan Date: Wed, 3 Jul 2013 14:54:42 -0700 Subject: [PATCH 05/10] add some test --- Makefile | 4 +-- test.lua | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 6713344..4191bcb 100644 --- a/Makefile +++ b/Makefile @@ -14,10 +14,10 @@ LUA_VERSION = 5.1 TARGET = cmsgpack.so PREFIX = /usr/local -CFLAGS = -g -Wall -pedantic -fno-inline +CFLAGS = -g -Wall -pedantic -fno-inline -fprofile-arcs -ftest-coverage -lgcov #CFLAGS = -O3 -Wall -pedantic -DNDEBUG CMSGPACK_CFLAGS = -fpic -CMSGPACK_LDFLAGS = -shared +CMSGPACK_LDFLAGS = -shared -fprofile-arcs LUA_INCLUDE_DIR = $(PREFIX)/include LUA_CMODULE_DIR = $(PREFIX)/lib/lua/$(LUA_VERSION) LUA_MODULE_DIR = $(PREFIX)/share/lua/$(LUA_VERSION) diff --git a/test.lua b/test.lua index 557678b..00cf284 100644 --- a/test.lua +++ b/test.lua @@ -4,7 +4,47 @@ require "cmsgpack" -- See the copyright notice at the end of lua_cmsgpack.c for more information. passed = 0 -failed = 0 +failed = 0 + +local Chars = {} +for Loop = 0, 255 do + Chars[Loop+1] = string.char(Loop) +end +local String = table.concat(Chars) + +local Built = {['.'] = Chars} + +local AddLookup = function(CharSet) + local Substitute = string.gsub(String, '[^'..CharSet..']', '') + local Lookup = {} + for Loop = 1, string.len(Substitute) do + Lookup[Loop] = string.sub(Substitute, Loop, Loop) + end + Built[CharSet] = Lookup + + return Lookup +end + +function string.random(Length, CharSet) + -- Length (number) + -- CharSet (string, optional); e.g. %l%d for lower case letters and digits + + local CharSet = CharSet or '.' + + if CharSet == '' then + return '' + else + local Result = {} + local Lookup = Built[CharSet] or AddLookup(CharSet) + local Range = table.getn(Lookup) + + for Loop = 1,Length do + Result[Loop] = Lookup[math.random(1, Range)] + end + + return table.concat(Result) + end +end function hex(s) local i @@ -118,7 +158,46 @@ test_circular("string16","xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa test_circular("fix array (1)",{1,2,3,"foo"}) test_circular("fix array (2)",{}) test_circular("fix array (3)",{1,{},{}}) -test_circular("fix map",{a=5,b=10,c="string"}) + +gap_array = {1,2,3,4} +gap_array[7] = 2 +test_circular("fix array (4)", gap_array) + +test_circular("fix map (1)",{a=5,b=10,c="string"}) + +hash_num = 5 +arrary_num = 1000000 +large_table = {} +for i = 1, hash_num do + large_table[string.random(5, "%l%d%u")] = i +end +for i = 1, arrary_num do + large_table[i] = i + 10000000 +end +test_circular("fix large map (2)", large_table) + +gap_hash = {1,2,3,4} +gap_hash[7] = 2 +gap_hash["hello"] = 4 +test_circular("fix map(3)", gap_array) + +hash_num = 5 +arrary_num = 65530 +medium_table = {} +for i = 1, hash_num do + medium_table[string.random(5, "%l%d%u")] = i +end +for i = 1, arrary_num do + medium_table[i] = i + 10000000 +end +test_circular("fix large map (4)", medium_table) + +arrary_num = 1000000 +large_array = {} +for i = 1, arrary_num do + large_array[i] = i + 10000000 +end +test_circular("large array", large_array) -- The following test vectors are taken from the Javascript lib at: -- https://github.com/cuzic/MessagePack-JS/blob/master/test/test_pack.html From 6d95f74910f893739915cd844bde10883931b535 Mon Sep 17 00:00:00 2001 From: Guanlan Dai Date: Mon, 15 Jul 2013 16:11:41 -0700 Subject: [PATCH 06/10] code style fix --- lua_cmsgpack.c | 246 ++++++++++++++++++++++++------------------------- 1 file changed, 123 insertions(+), 123 deletions(-) diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index 7d87512..21a3bc0 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -69,7 +69,7 @@ typedef struct mp_buf { static mp_buf *mp_buf_new(void) { mp_buf *buf = malloc(sizeof(*buf)); - + buf->b = NULL; buf->len = buf->free = 0; return buf; @@ -372,7 +372,7 @@ static int table_is_an_array(lua_State *L) { lua_pushnil(L); while(lua_next(L,-2)) { - idx++; + idx++; /* Stack: ... key value */ lua_pop(L,1); /* Stack: ... key */ if (!lua_isnumber(L,-1)) goto not_array; @@ -435,128 +435,128 @@ static int mp_pack(lua_State *L) { /* --------------------------------- Decoding --------------------------------- */ -unsigned int next_power_of_two(int n) -{ - n--; - n |= n >> 1; - n |= n >> 2; - n |= n >> 4; - n |= n >> 8; - n |= n >> 16; - n++; - return n; +unsigned int next_power_of_two(int n) { + n--; + n |= n >> 1; + n |= n >> 2; + n |= n >> 4; + n |= n >> 8; + n |= n >> 16; + n++; + return n; } int calc_array_num(mp_cur *c, size_t len) { - const unsigned char *curr = c->p; - size_t left = c->left; - int narray = 0; - int i; - int idx = 0; - int value; - for (i = 0; i < len * 2; i++) { - switch (curr[0]) { - case 0xcc: /* uint 8 */ - case 0xd0: /* int 8 */ - if (i % 2 == 0) { - idx++; - if (left < 2) { - return narray; - } - - if (idx == curr[1] || curr[1] <= next_power_of_two(idx)) - narray++; - else - return narray; - } - curr += 2; - left -=2; - break; - case 0xcd: /* uint 16 */ - case 0xd1: /* int 16 */ - if (i % 2 == 0) { - idx++; - if (left < 3) { - return narray; - } - value = (curr[1] << 8) | curr[2]; - if (idx == value || value <= next_power_of_two(idx)) - narray++; - else - return narray; - } - curr += 3; - left -= 3; - break; - case 0xce: /* uint 32 */ - case 0xd2: /* int 32 */ - if (i % 2 == 0) { - idx++; - if (left < 5) { - return narray; - } - value = ((uint32_t)curr[1] << 24) | - ((uint32_t)curr[2] << 16) | - ((uint32_t)curr[3] << 8) | - (uint32_t)curr[4]; - if (idx == value || value <= next_power_of_two(idx)) - narray++; - else - return narray; - } - curr += 5; - left -= 5; - break; - case 0xcf: /* uint 64 */ - case 0xd3: /* int 64 */ - if (i % 2 == 0) { - idx++; - if (left < 9) { - return narray; - } - - value = ((uint64_t)curr[1] << 56) | - ((uint64_t)curr[2] << 48) | - ((uint64_t)curr[3] << 40) | - ((uint64_t)curr[4] << 32) | - ((uint64_t)curr[5] << 24) | - ((uint64_t)curr[6] << 16) | - ((uint64_t)curr[7] << 8) | - (uint64_t)curr[8]; - if (idx == value || value <= next_power_of_two(idx)) - narray++; - else - return narray; - } - curr += 9; - left -= 9; - break; - default: /* types that can't be idenitified by first byte value. */ - if ((curr[0] & 0x80) == 0) { /* positive fixnum */ - if (i % 2 == 0) { - idx++; - if (idx == curr[0] || curr[0] <= next_power_of_two(idx)) { - narray++; - } else { - return narray; - } - } - curr += 1; - left -= 1; - } else { /* other */ - return narray; - } - } - } - return narray; + const unsigned char *curr = c->p; + size_t left = c->left; + int narray = 0; + int i; + int idx = 0; + int value; + for (i = 0; i < len * 2; i++) { + switch (curr[0]) { + case 0xcc: /* uint 8 */ + case 0xd0: /* int 8 */ + if (i % 2 == 0) { + idx++; + if (left < 2) { + return narray; + } + + if (idx == curr[1] || curr[1] <= next_power_of_two(idx)) + narray++; + else + return narray; + } + curr += 2; + left -=2; + break; + case 0xcd: /* uint 16 */ + case 0xd1: /* int 16 */ + if (i % 2 == 0) { + idx++; + if (left < 3) { + return narray; + } + value = (curr[1] << 8) | curr[2]; + if (idx == value || value <= next_power_of_two(idx)) + narray++; + else + return narray; + } + curr += 3; + left -= 3; + break; + case 0xce: /* uint 32 */ + case 0xd2: /* int 32 */ + if (i % 2 == 0) { + idx++; + if (left < 5) { + return narray; + } + value = ((uint32_t)curr[1] << 24) | + ((uint32_t)curr[2] << 16) | + ((uint32_t)curr[3] << 8) | + (uint32_t)curr[4]; + if (idx == value || value <= next_power_of_two(idx)) + narray++; + else + return narray; + } + curr += 5; + left -= 5; + break; + case 0xcf: /* uint 64 */ + case 0xd3: /* int 64 */ + if (i % 2 == 0) { + idx++; + if (left < 9) { + return narray; + } + + value = ((uint64_t)curr[1] << 56) | + ((uint64_t)curr[2] << 48) | + ((uint64_t)curr[3] << 40) | + ((uint64_t)curr[4] << 32) | + ((uint64_t)curr[5] << 24) | + ((uint64_t)curr[6] << 16) | + ((uint64_t)curr[7] << 8) | + (uint64_t)curr[8]; + if (idx == value || value <= next_power_of_two(idx)) + narray++; + else + return narray; + } + curr += 9; + left -= 9; + break; + default: /* types that can't be idenitified by first byte value. */ + if ((curr[0] & 0x80) == 0) { /* positive fixnum */ + if (i % 2 == 0) { + idx++; + if (idx == curr[0] || curr[0] <= next_power_of_two(idx)) { + narray++; + } else { + return narray; + } + } + curr += 1; + left -= 1; + + } else { /* other */ + return narray; + } + } + } + return narray; } void mp_decode_to_lua_type(lua_State *L, mp_cur *c); void mp_decode_to_lua_array(lua_State *L, mp_cur *c, size_t len) { int index = 1; - /*lua_newtable(L);*/ - lua_createtable(L, len, 0); + /*lua_newtable(L);*/ + lua_createtable(L, len, 0); while(len--) { lua_pushnumber(L,index++); mp_decode_to_lua_type(L,c); @@ -566,12 +566,12 @@ void mp_decode_to_lua_array(lua_State *L, mp_cur *c, size_t len) { } void mp_decode_to_lua_hash(lua_State *L, mp_cur *c, size_t len) { - int narray; - int nhash; - narray = calc_array_num(c, len); - nhash = len - narray; - lua_createtable(L, narray, nhash); - /*lua_newtable(L);*/ + int narray; + int nhash; + narray = calc_array_num(c, len); + nhash = len - narray; + lua_createtable(L, narray, nhash); + /*lua_newtable(L);*/ while(len--) { mp_decode_to_lua_type(L,c); /* key */ if (c->err) return; @@ -786,7 +786,7 @@ static int mp_unpack(lua_State *L) { s = (const unsigned char*) lua_tolstring(L,-1,&len); c = mp_cur_new(s,len); mp_decode_to_lua_type(L,c); - + if (c->err == MP_CUR_ERROR_EOF) { mp_cur_free(c); lua_pushstring(L,"Missing bytes in input."); @@ -820,7 +820,7 @@ LUALIB_API int luaopen_cmsgpack (lua_State *L) { lua_pushliteral(L, LUACMSGPACK_COPYRIGHT); lua_setfield(L, -2, "_COPYRIGHT"); lua_pushliteral(L, LUACMSGPACK_DESCRIPTION); - lua_setfield(L, -2, "_DESCRIPTION"); + lua_setfield(L, -2, "_DESCRIPTION"); return 1; } From 069cb8557cac56198d7585293a61673a0a1c19e4 Mon Sep 17 00:00:00 2001 From: Guanlan Dai Date: Mon, 15 Jul 2013 18:02:00 -0700 Subject: [PATCH 07/10] throw a exception when exceed the max nesting --- Makefile | 3 ++- lua_cmsgpack.c | 9 +++++++-- test.lua | 28 ++++++++++++++++++---------- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 4191bcb..31e5b16 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,8 @@ LUA_VERSION = 5.1 TARGET = cmsgpack.so PREFIX = /usr/local -CFLAGS = -g -Wall -pedantic -fno-inline -fprofile-arcs -ftest-coverage -lgcov +CFLAGS = -g -O3 -Wall -pedantic -DNDEBUG -DLUA_USE_APICHECK +#CFLAGS = -g -Wall -pedantic -fno-inline -fprofile-arcs -ftest-coverage -lgcov #CFLAGS = -O3 -Wall -pedantic -DNDEBUG CMSGPACK_CFLAGS = -fpic CMSGPACK_LDFLAGS = -shared -fprofile-arcs diff --git a/lua_cmsgpack.c b/lua_cmsgpack.c index 21a3bc0..68239ce 100644 --- a/lua_cmsgpack.c +++ b/lua_cmsgpack.c @@ -11,7 +11,7 @@ #define LUACMSGPACK_COPYRIGHT "Copyright (C) 2012, Salvatore Sanfilippo" #define LUACMSGPACK_DESCRIPTION "MessagePack C implementation for Lua" -#define LUACMSGPACK_MAX_NESTING 16 /* Max tables nesting. */ +#define LUACMSGPACK_MAX_NESTING 9 /* Max tables nesting. */ /* ============================================================================== * MessagePack implementation and bindings for Lua 5.1. @@ -413,7 +413,12 @@ static void mp_encode_lua_type(lua_State *L, mp_buf *buf, int level) { /* Limit the encoding of nested tables to a specfiied maximum depth, so that * we survive when called against circular references in tables. */ - if (t == LUA_TTABLE && level == LUACMSGPACK_MAX_NESTING) t = LUA_TNIL; + if (t == LUA_TTABLE && level == LUACMSGPACK_MAX_NESTING) { + t = LUA_TNIL; + lua_pushliteral(L, "table nesting level too deep"); + lua_error(L); + } + switch(t) { case LUA_TSTRING: mp_encode_lua_string(L,buf); break; case LUA_TBOOLEAN: mp_encode_lua_bool(L,buf); break; diff --git a/test.lua b/test.lua index 00cf284..80db891 100644 --- a/test.lua +++ b/test.lua @@ -169,10 +169,10 @@ hash_num = 5 arrary_num = 1000000 large_table = {} for i = 1, hash_num do - large_table[string.random(5, "%l%d%u")] = i + large_table[string.random(5, "%l%d%u")] = i end for i = 1, arrary_num do - large_table[i] = i + 10000000 + large_table[i] = i + 10000000 end test_circular("fix large map (2)", large_table) @@ -185,17 +185,17 @@ hash_num = 5 arrary_num = 65530 medium_table = {} for i = 1, hash_num do - medium_table[string.random(5, "%l%d%u")] = i + medium_table[string.random(5, "%l%d%u")] = i end for i = 1, arrary_num do - medium_table[i] = i + 10000000 + medium_table[i] = i + 10000000 end test_circular("fix large map (4)", medium_table) arrary_num = 1000000 large_array = {} for i = 1, arrary_num do - large_array[i] = i + 10000000 + large_array[i] = i + 10000000 end test_circular("large array", large_array) @@ -222,11 +222,19 @@ test_pack_and_unpack("raw16"," ","da00282 test_pack_and_unpack("array 16",{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},"dc001000000000000000000000000000000000") -- Regression test for issue #4, cyclic references in tables. -a = {x=nil,y=5} -b = {x=a} -a['x'] = b -pack = cmsgpack.pack(a) -test_pack("regression for issue #4",a,"82a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a178c0") +function cyclic_refer_test() + a = {x=nil,y=5} + b = {x=a} + a['x'] = b + cmsgpack.pack(a) +end + +if pcall(cyclic_refer_test) then + print "FAILED: exception no throw" +else + print "Testing max nesting ...ok" +end +-- test_pack("regression for issue #4",a,"82a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a178c0") -- Final report print() From e0f97c860a3a1e13a0d67677490a816fbbb07f98 Mon Sep 17 00:00:00 2001 From: Guanlan Dai Date: Tue, 16 Jul 2013 11:47:42 -0700 Subject: [PATCH 08/10] makefile flag changes --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 31e5b16..b751ae3 100644 --- a/Makefile +++ b/Makefile @@ -14,11 +14,12 @@ LUA_VERSION = 5.1 TARGET = cmsgpack.so PREFIX = /usr/local -CFLAGS = -g -O3 -Wall -pedantic -DNDEBUG -DLUA_USE_APICHECK +CFLAGS = -g -O3 -Wall -pedantic #CFLAGS = -g -Wall -pedantic -fno-inline -fprofile-arcs -ftest-coverage -lgcov #CFLAGS = -O3 -Wall -pedantic -DNDEBUG CMSGPACK_CFLAGS = -fpic -CMSGPACK_LDFLAGS = -shared -fprofile-arcs +CMSGPACK_LDFLAGS = -shared +#CMSGPACK_LDFLAGS = -shared -fprofile-arcs LUA_INCLUDE_DIR = $(PREFIX)/include LUA_CMODULE_DIR = $(PREFIX)/lib/lua/$(LUA_VERSION) LUA_MODULE_DIR = $(PREFIX)/share/lua/$(LUA_VERSION) From b84be41685442c70dad6f53a652e91d68722d485 Mon Sep 17 00:00:00 2001 From: Guanlan Dai Date: Tue, 16 Jul 2013 12:57:33 -0700 Subject: [PATCH 09/10] Update readme --- README | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/README b/README index d6317b1..e33ddd5 100644 --- a/README +++ b/README @@ -10,20 +10,9 @@ The library is currently considered in BETA STAGE for lack of extensive testing. INSTALLATION --- -Using LuaRocks (http://luarocks.org): + make + sudo make install -* Install current stable release: - - sudo luarocks install lua-cmsgpack - -* Install current Git master head from GitHub: - - sudo luarocks install lua-cmsgpack --from=rocks-cvs - -* Install from current working copy - - cd lua-cmsgpack/ - sudo luarocks make rockspec/lua-cmsgpack-scm-1.rockspec If you embed Lua and all modules into your C project, just add the lua_cmsgpack.c file and call the following function after creating the Lua @@ -54,9 +43,9 @@ maps. NESTED TABLES --- Nested tables are handled correctly up to LUACMSGPACK_MAX_NESTING levels of -nesting (that is set to 16 by default). +nesting (that is set to 9 by default). Every table that is nested at a greater level than the maxium is encoded -as MessagePack nil value. +as MessagePack nil value and throw a exception. It is worth to note that in Lua it is possible to create tables that mutually refer to each other, creating a cycle. For example: @@ -65,8 +54,7 @@ a = {x=nil,y=5} b = {x=a} a['x'] = b -This condition will simply make the encoder reach the max level of nesting, -thus avoiding an infinite loop. +This condition will throw a exception. CREDITS --- From f5b838ac552cbada957749d6938cd96a4e7a1f62 Mon Sep 17 00:00:00 2001 From: Guanlan Dai Date: Wed, 17 Jul 2013 12:39:40 -0700 Subject: [PATCH 10/10] Update README --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README b/README index e33ddd5..4c4fe2b 100644 --- a/README +++ b/README @@ -10,7 +10,7 @@ The library is currently considered in BETA STAGE for lack of extensive testing. INSTALLATION --- - make + make sudo make install