From 1d43eb97005bd46db6d9865b29140a58a1c298dc Mon Sep 17 00:00:00 2001 From: Steve Balbach Date: Mon, 8 Jul 2019 09:05:56 -0500 Subject: [PATCH 01/13] add support for terraform 0.12 --- CHANGELOG.md | 4 ++ src/hcl/lexer.py | 44 ++++++++++++++++ src/hcl/parser.py | 121 +++++++++++++++++++++++++++++++++++++++----- tests/test_lexer.py | 4 +- 4 files changed, 157 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20bf52f..ef9791d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +0.3.13 (2019-06-25) +------------------ +* Updated for terraform v0.12 syntax + 0.2.1 (2016-04-14) ------------------ * Fixes for mutliple configuration blocks with same name diff --git a/src/hcl/lexer.py b/src/hcl/lexer.py index 176d7dc..9606d16 100644 --- a/src/hcl/lexer.py +++ b/src/hcl/lexer.py @@ -44,6 +44,16 @@ class Lexer(object): 'PERIOD', 'EPLUS', 'EMINUS', + 'LEFTPAREN', + 'RIGHTPAREN', + 'QMARK', + 'COLON', + 'GT', + 'LT', + 'EQ', + 'NE', + 'LE', + 'GE', ) states = ( @@ -91,6 +101,38 @@ def t_COMMA(self, t): r',' return t + def t_QMARK(self, t): + r'\?' + return t + + def t_COLON(self, t): + r':' + return t + + def t_GT(self, t): + r'>' + return t + + def t_LT(self, t): + r'<[^<]' + return t + + def t_EQ(self, t): + r'==' + return t + + def t_NE(self, t): + r'!=' + return t + + def t_LE(self, t): + r'<=' + return t + + def t_GE(self, t): + r'>=' + return t + def t_IDENTIFIER(self, t): r'[^\W\d][\w.-]*' t.value = text_type(t.value) @@ -266,6 +308,8 @@ def t_heredoc_eof(self, t): t_RIGHTBRACE = r'\}' t_LEFTBRACKET = r'\[' t_RIGHTBRACKET = r'\]' + t_LEFTPAREN = r'\(' + t_RIGHTPAREN = r'\)' def t_COMMENT(self, t): r'(\#|(//)).*' diff --git a/src/hcl/parser.py b/src/hcl/parser.py index 444497d..7998d23 100644 --- a/src/hcl/parser.py +++ b/src/hcl/parser.py @@ -57,6 +57,16 @@ class HclParser(object): 'PERIOD', 'EPLUS', 'EMINUS', + 'LEFTPAREN', + 'RIGHTPAREN', + 'QMARK', + 'COLON', + 'GT', + 'LT', + 'EQ', + 'NE', + 'LE', + 'GE', ) # @@ -100,7 +110,7 @@ def objectlist_flat(self, lt, replace): d[k] = v return d - + def p_top(self, p): "top : objectlist" if DEBUG: @@ -157,6 +167,7 @@ def p_objectitem_0(self, p): objectitem : objectkey EQUAL number | objectkey EQUAL BOOL | objectkey EQUAL STRING + | objectkey EQUAL IDENTIFIER | objectkey EQUAL object | objectkey EQUAL list ''' @@ -170,31 +181,78 @@ def p_objectitem_1(self, p): self.print_p(p) p[0] = p[1] - def p_block_0(self, p): - "block : blockId object" + def p_objectitem_2(self, p): + ''' + objectitem : objectkey EQUAL IDENTIFIER LEFTBRACKET IDENTIFIER RIGHTBRACKET + ''' if DEBUG: self.print_p(p) - p[0] = (p[1], p[2]) - def p_block_1(self, p): - "block : blockId block" + p[0] = (p[1], p[3] + p[4] + p[5] + p[6]) + + def p_objectitem_3(self, p): + ''' + objectitem : objectkey EQUAL IDENTIFIER list + ''' if DEBUG: self.print_p(p) - p[0] = (p[1], {p[2][0]: p[2][1]}) - def p_blockId(self, p): + p[4].insert(0, p[3]) + p[0] = (p[1], p[4]) + + def p_objectitem_4(self, p): ''' - blockId : IDENTIFIER - | STRING + objectitem : objectkey EQUAL objectkey QMARK objectkey COLON objectkey + | objectkey EQUAL objectkey QMARK objectkey COLON number + | objectkey EQUAL objectkey QMARK number COLON objectkey + | objectkey EQUAL objectkey QMARK number COLON number + | objectkey EQUAL booleanexp QMARK objectkey COLON objectkey + | objectkey EQUAL booleanexp QMARK objectkey COLON number + | objectkey EQUAL booleanexp QMARK number COLON objectkey + | objectkey EQUAL booleanexp QMARK number COLON number + ''' + if DEBUG: + self.print_p(p) + p[0] = (p[1], p[3] + p[4] + str(p[5]) + p[6] + str(p[7])) + + def p_operator_0(self, p): + ''' + operator : EQ + | NE + | LT + | GT + | LE + | GE ''' if DEBUG: self.print_p(p) p[0] = p[1] + def p_booleanexp_0(self, p): + "booleanexp : objectkey operator objectkey" + "booleanexp : objectkey operator number" + "booleanexp : number operator objectkey" + if DEBUG: + self.print_p(p) + p[0] = str(p[1]) + p[2] + str(p[3]) + + def p_block_0(self, p): + "block : objectkey object" + if DEBUG: + self.print_p(p) + p[0] = (p[1], p[2]) + + def p_block_1(self, p): + "block : objectkey block" + if DEBUG: + self.print_p(p) + p[0] = (p[1], {p[2][0]: p[2][1]}) + def p_list_0(self, p): ''' list : LEFTBRACKET listitems RIGHTBRACKET | LEFTBRACKET listitems COMMA RIGHTBRACKET + | LEFTPAREN listitems RIGHTPAREN ''' if DEBUG: self.print_p(p) @@ -207,27 +265,62 @@ def p_list_1(self, p): p[0] = [] def p_listitems_0(self, p): - "listitems : listitem" + ''' + listitems : listitem + | object COMMA + | objectkey COMMA + ''' if DEBUG: self.print_p(p) p[0] = [p[1]] def p_listitems_1(self, p): - "listitems : listitems COMMA listitem" + ''' + listitems : listitems COMMA listitem + | listitems COMMA objectkey + ''' if DEBUG: self.print_p(p) p[0] = p[1] + [p[3]] - def p_listitem(self, p): + def p_listitems_2(self, p): + ''' + listitems : object COMMA object + | object COMMA objectkey + | objectkey COMMA objectkey + | objectkey COMMA object + ''' + if DEBUG: + self.print_p(p) + p[0] = [p[1], p[3]] + + def p_listitems_3(self, p): + ''' + listitems : objectkey list + ''' + if DEBUG: + self.print_p(p) + p[2].insert(0, p[1]) + p[0] = p[2] + + def p_listitem_0(self, p): ''' listitem : number | object - | STRING + | objectkey ''' if DEBUG: self.print_p(p) p[0] = p[1] + def p_listitem_1(self, p): + ''' + listitem : objectkey LEFTBRACKET objectkey RIGHTBRACKET + ''' + if DEBUG: + self.print_p(p) + p[0] = (p[1] + p[2] + p[3] + p[4]) + def p_number_0(self, p): "number : int" if DEBUG: diff --git a/tests/test_lexer.py b/tests/test_lexer.py index 6242188..1d6cb45 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -106,7 +106,7 @@ class Error: ( "old.hcl", [ - "IDENTIFIER", "EQUAL", "LEFTBRACE", "STRING", Error + "IDENTIFIER", "EQUAL", "LEFTBRACE", "STRING", "COLON", "STRING" ], "Line 2, column 15, index 27: Illegal character ':'" ), @@ -129,7 +129,7 @@ class Error: "a = Date: Thu, 18 Jul 2019 11:06:54 -0500 Subject: [PATCH 02/13] More terraform 0.12 syntax updates; also added tests for the terraform 0.12 syntax updates --- src/hcl/lexer.py | 13 ++- src/hcl/parser.py | 96 ++++++++++++++++------ tests/lex-fixtures/terraform0.12syntax.hcl | 33 ++++++++ tests/test_lexer.py | 12 +++ tests/test_parser.py | 6 +- 5 files changed, 129 insertions(+), 31 deletions(-) create mode 100644 tests/lex-fixtures/terraform0.12syntax.hcl diff --git a/src/hcl/lexer.py b/src/hcl/lexer.py index 9606d16..ac671ae 100644 --- a/src/hcl/lexer.py +++ b/src/hcl/lexer.py @@ -48,6 +48,7 @@ class Lexer(object): 'RIGHTPAREN', 'QMARK', 'COLON', + 'ASTERISK_PERIOD', 'GT', 'LT', 'EQ', @@ -73,7 +74,7 @@ def t_EMINUS(self, t): return t def t_EPLUS(self, t): - r'(?<=\d|\.)[eE]\+?' + r'(?<=\d)[eE]\+?|(?<=\d\.)[eE]\+?' return t def t_FLOAT(self, t): @@ -109,12 +110,16 @@ def t_COLON(self, t): r':' return t + def t_ASTERISK_PERIOD(self, t): + r'\*\.' + return t + def t_GT(self, t): - r'>' + r'(?)>(?!>|=)' return t def t_LT(self, t): - r'<[^<]' + r'(?"), + ("LT", "<"), + ("EQ", "=="), + ("NE", "!="), + ("LE", "<="), + ("GE", ">="), + ("ASTERISK_PERIOD", "*."), # Bools ("BOOL", "true"), @@ -382,6 +393,7 @@ def test_tokens(token, input_string): # other token COMPLEX_TOKEN_FIXTURES = [ # EPLUS + (["IDENTIFIER", "EQUAL", "LEFTBRACKET", "IDENTIFIER", "LEFTBRACKET", "NUMBER", "RIGHTBRACKET", "PERIOD", "IDENTIFIER", "RIGHTBRACKET", ], "records = [aws_elasticsearch_domain.elasticsearch[0].endpoint]"), (["FLOAT", "EPLUS"], "0.e"), (["FLOAT", "EPLUS"], "1.e+"), (["NUMBER", "EPLUS"], "0e"), diff --git a/tests/test_parser.py b/tests/test_parser.py index 56e3b71..eb63386 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -14,7 +14,7 @@ PARSE_FIXTURES = [ ( "assign_colon.hcl", - True, + False, ), ( "comment.hcl", @@ -60,6 +60,10 @@ "structure_comma.hcl", False, ), + ( + "terraform0.12syntax.hcl", + False, + ), ] @pytest.mark.parametrize("hcl_fname,invalid", PARSE_FIXTURES) From fe1127eff8927d45ca32527ad6b99f50261ea17f Mon Sep 17 00:00:00 2001 From: Steve Balbach Date: Wed, 24 Jul 2019 14:58:45 -0500 Subject: [PATCH 03/13] more terraform 0.12 syntax updates --- src/hcl/parser.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/hcl/parser.py b/src/hcl/parser.py index 7d45442..bded5fb 100644 --- a/src/hcl/parser.py +++ b/src/hcl/parser.py @@ -188,6 +188,7 @@ def p_objectitem_0(self, p): | objectkey EQUAL list | objectkey EQUAL objectbrackets | objectkey EQUAL function + | objectkey EQUAL booleanexp | objectkey COLON number | objectkey COLON BOOL | objectkey COLON STRING @@ -195,6 +196,7 @@ def p_objectitem_0(self, p): | objectkey COLON object | objectkey COLON list | objectkey COLON objectbrackets + | objectkey COLON booleanexp ''' if DEBUG: self.print_p(p) @@ -292,7 +294,7 @@ def p_list_2(self, p): ''' if DEBUG: self.print_p(p) - p[0] = p[1] + p[2] + p[3] + p[4] + p[5] + p[6] + p[7] + p[8] + p[0] = [p[3]] + [p[5] + p[6] + p[7]] def p_function_0(self, p): ''' From 39ef24238a537b8c32b5e7a6183408fabca97bcb Mon Sep 17 00:00:00 2001 From: Steve Balbach Date: Mon, 29 Jul 2019 15:13:59 -0500 Subject: [PATCH 04/13] add support for arithmetic operators --- src/hcl/lexer.py | 14 ++++++++---- src/hcl/parser.py | 52 +++++++++++++++++++++++++++++++++++++++++++++ tests/test_lexer.py | 51 +++++++++++++++++++++++--------------------- 3 files changed, 89 insertions(+), 28 deletions(-) diff --git a/src/hcl/lexer.py b/src/hcl/lexer.py index ac671ae..287b4e8 100644 --- a/src/hcl/lexer.py +++ b/src/hcl/lexer.py @@ -36,7 +36,10 @@ class Lexer(object): 'IDENTIFIER', 'EQUAL', 'STRING', + 'ADD', 'MINUS', + 'MULTIPLY', + 'DIVIDE', 'LEFTBRACE', 'RIGHTBRACE', 'LEFTBRACKET', @@ -306,9 +309,6 @@ def t_heredoc_eof(self, t): t_tabbedheredoc_ignoring = t_heredoc_ignoring t_tabbedheredoc_eof = t_heredoc_eof - t_EQUAL = r'(?"), From 8cccfc74ef5f0f3df5eb12904ebd6392130cf75b Mon Sep 17 00:00:00 2001 From: Steve Balbach Date: Mon, 29 Jul 2019 15:30:28 -0500 Subject: [PATCH 05/13] update for black formatting --- src/hcl/lexer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hcl/lexer.py b/src/hcl/lexer.py index 287b4e8..de02865 100644 --- a/src/hcl/lexer.py +++ b/src/hcl/lexer.py @@ -331,13 +331,13 @@ def t_newline(self, t): t.lexer.lineno += len(t.value) t_ignore = ' \t\r\f\v' - + t_EQUAL = r'(? Date: Wed, 31 Jul 2019 11:28:28 -0500 Subject: [PATCH 06/13] Allow for functions in Conditional expressions (all permutations of objectkey EQUAL objectkey QMARK objectkey COLON function). Also add tests for these situations. --- src/hcl/parser.py | 12 ++++- tests/lex-fixtures/conditional_operator.hcl | 53 +++++++++++++++++++++ tests/test_parser.py | 4 ++ 3 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 tests/lex-fixtures/conditional_operator.hcl diff --git a/src/hcl/parser.py b/src/hcl/parser.py index f90874c..93f82ce 100644 --- a/src/hcl/parser.py +++ b/src/hcl/parser.py @@ -233,20 +233,28 @@ def p_objectitem_2(self, p): objectitem : objectkey EQUAL objectkey QMARK objectkey COLON objectkey | objectkey EQUAL objectkey QMARK objectkey COLON number | objectkey EQUAL objectkey QMARK objectkey COLON BOOL - | objectkey EQUAL objectkey QMARK BOOL COLON objectkey + | objectkey EQUAL objectkey QMARK objectkey COLON function | objectkey EQUAL objectkey QMARK number COLON objectkey + | objectkey EQUAL objectkey QMARK BOOL COLON objectkey + | objectkey EQUAL objectkey QMARK function COLON objectkey | objectkey EQUAL objectkey QMARK number COLON number | objectkey EQUAL objectkey QMARK number COLON BOOL + | objectkey EQUAL objectkey QMARK number COLON function | objectkey EQUAL objectkey QMARK BOOL COLON number + | objectkey EQUAL objectkey QMARK BOOL COLON function | objectkey EQUAL objectkey QMARK BOOL COLON BOOL | objectkey EQUAL booleanexp QMARK objectkey COLON objectkey | objectkey EQUAL booleanexp QMARK objectkey COLON number | objectkey EQUAL booleanexp QMARK objectkey COLON BOOL - | objectkey EQUAL booleanexp QMARK BOOL COLON objectkey + | objectkey EQUAL booleanexp QMARK objectkey COLON function | objectkey EQUAL booleanexp QMARK number COLON objectkey + | objectkey EQUAL booleanexp QMARK BOOL COLON objectkey + | objectkey EQUAL booleanexp QMARK function COLON objectkey | objectkey EQUAL booleanexp QMARK number COLON number | objectkey EQUAL booleanexp QMARK number COLON BOOL + | objectkey EQUAL booleanexp QMARK number COLON function | objectkey EQUAL booleanexp QMARK BOOL COLON number + | objectkey EQUAL booleanexp QMARK BOOL COLON function | objectkey EQUAL booleanexp QMARK BOOL COLON BOOL ''' if DEBUG: diff --git a/tests/lex-fixtures/conditional_operator.hcl b/tests/lex-fixtures/conditional_operator.hcl new file mode 100644 index 0000000..bb9e8d5 --- /dev/null +++ b/tests/lex-fixtures/conditional_operator.hcl @@ -0,0 +1,53 @@ +// objectitem : objectkey EQUAL objectkey QMARK objectkey COLON objectkey +identifier1 = identifier2 ? identifier3 : identifier4 +// objectitem : objectkey EQUAL objectkey QMARK objectkey COLON number +identifier1 = identifier2 ? identifier3 : 1 +// objectitem : objectkey EQUAL objectkey QMARK objectkey COLON BOOL +identifier1 = identifier2 ? identifier3 : True +// objectitem : objectkey EQUAL objectkey QMARK objectkey COLON function +identifier1 = identifier2 ? identifier3 : element(identifier4, identifier5) +// objectitem : objectkey EQUAL objectkey QMARK number COLON objectkey +identifier1 = identifier2 ? 1 : identifier3 +// objectitem : objectkey EQUAL objectkey QMARK BOOL COLON objectkey +identifier1 = identifier2 ? True : identifier3 +// objectitem : objectkey EQUAL objectkey QMARK function COLON objectkey +identifier1 = identifier2 ? element(identifier3, identifier4) : identifier5 +// objectitem : objectkey EQUAL objectkey QMARK number COLON number +identifier1 = identifier2 ? 1 : 2 +// objectitem : objectkey EQUAL objectkey QMARK number COLON BOOL +identifier1 = identifier2 ? 1 : True +// objectitem : objectkey EQUAL objectkey QMARK number COLON function +identifier1 = identifier2 ? 1 : element(identifier3, identifier4) +// objectitem : objectkey EQUAL objectkey QMARK BOOL COLON number +identifier1 = identifier2 ? True : 1 +// objectitem : objectkey EQUAL objectkey QMARK BOOL COLON function +identifier1 = identifier2 ? True : element(identifier3, identifier4) +// objectitem : objectkey EQUAL objectkey QMARK BOOL COLON BOOL +identifier1 = identifier2 ? True : False + +// objectitem : objectkey EQUAL booleanexp QMARK objectkey COLON objectkey +identifier1 = identifier2 == identifier3 ? identifier4 : identifier5 +// objectitem : objectkey EQUAL booleanexp QMARK objectkey COLON number +identifier1 = identifier2 == identifier3 ? identifier4 : 1 +// objectitem : objectkey EQUAL booleanexp QMARK objectkey COLON BOOL +identifier1 = identifier2 == identifier3 ? identifier4 : True +// objectitem : objectkey EQUAL booleanexp QMARK objectkey COLON function +identifier1 = identifier2 == identifier3 ? identifier4 : element(identifier5, identifier6) +// objectitem : objectkey EQUAL booleanexp QMARK number COLON objectkey +identifier1 = identifier2 == identifier3 ? 1 : identifier4 +// objectitem : objectkey EQUAL booleanexp QMARK BOOL COLON objectkey +identifier1 = identifier2 == identifier3 ? True : identifier4 +// objectitem : objectkey EQUAL booleanexp QMARK function COLON objectkey +identifier1 = identifier2 == identifier3 ? element(identifier4, identifier5) : identifier6 +// objectitem : objectkey EQUAL booleanexp QMARK number COLON number +identifier1 = identifier2 == identifier3 ? 1 : 2 +// objectitem : objectkey EQUAL booleanexp QMARK number COLON BOOL +identifier1 = identifier2 == identifier3 ? 1 : True +// objectitem : objectkey EQUAL booleanexp QMARK number COLON function +identifier1 = identifier2 == identifier3 ? 1 : element(identifier4, identifier5) +// objectitem : objectkey EQUAL booleanexp QMARK BOOL COLON number +identifier1 = identifier2 == identifier3 ? True : 1 +// objectitem : objectkey EQUAL booleanexp QMARK BOOL COLON function +identifier1 = identifier2 == identifier3 ? True : element(identifier4, identifier5) +// objectitem : objectkey EQUAL booleanexp QMARK BOOL COLON BOOL +identifier1 = identifier2 == identifier3 ? True : False \ No newline at end of file diff --git a/tests/test_parser.py b/tests/test_parser.py index eb63386..5984541 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -64,6 +64,10 @@ "terraform0.12syntax.hcl", False, ), + ( + "conditional_operator.hcl", + False, + ), ] @pytest.mark.parametrize("hcl_fname,invalid", PARSE_FIXTURES) From 0c03c18c77badb4118a67fc9551c04bde78ea3ec Mon Sep 17 00:00:00 2001 From: Steve Balbach Date: Mon, 5 Aug 2019 12:46:25 -0500 Subject: [PATCH 07/13] change how functions are parsed --- src/hcl/parser.py | 49 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/src/hcl/parser.py b/src/hcl/parser.py index 93f82ce..5fe902f 100644 --- a/src/hcl/parser.py +++ b/src/hcl/parser.py @@ -300,8 +300,6 @@ def p_list_0(self, p): ''' list : LEFTBRACKET listitems RIGHTBRACKET | LEFTBRACKET listitems COMMA RIGHTBRACKET - | LEFTPAREN listitems RIGHTPAREN - | LEFTPAREN listitems COMMA RIGHTPAREN ''' if DEBUG: self.print_p(p) @@ -326,12 +324,53 @@ def p_list_2(self, p): def p_function_0(self, p): ''' - function : IDENTIFIER list + function : IDENTIFIER LEFTPAREN listitems RIGHTPAREN ''' if DEBUG: self.print_p(p) - p[2].insert(0, p[1]) - p[0] = p[2] + + p[0] = p[1] + p[2] + self.flatten(p[3]) + p[4] + + def p_function_1(self, p): + ''' + function : IDENTIFIER LEFTPAREN listitems COMMA RIGHTPAREN + ''' + if DEBUG: + self.print_p(p) + + p[0] = p[1] + p[2] + self.flatten(p[3]) + p[5] + + def flatten(self, value): + returnValue = "" + if type(value) is dict: + returnValue += "{" + isFirstTime = True + for key in value: + if isFirstTime: + isFirstTime = False + else: + returnValue += "," + returnValue += key + ":" + self.flatten(value[key]) + returnValue += "}" + elif type(value) is list: + isFirstTime = True + for v in value: + if isFirstTime: + isFirstTime = False + else: + returnValue += "," + returnValue += self.flatten(v) + elif type(value) is tuple: + isFirstTime = True + for v in value: + if isFirstTime: + isFirstTime = False + else: + returnValue += "," + returnValue += self.flatten(v) + else: + returnValue = value + return returnValue def p_listitems_0(self, p): ''' From 8767f0801317fc1f64e6f628ba81ccc70fb4870d Mon Sep 17 00:00:00 2001 From: Steve Balbach Date: Mon, 5 Aug 2019 13:12:37 -0500 Subject: [PATCH 08/13] change formatting to make black happy --- src/hcl/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hcl/parser.py b/src/hcl/parser.py index 5fe902f..4ae6e14 100644 --- a/src/hcl/parser.py +++ b/src/hcl/parser.py @@ -330,7 +330,7 @@ def p_function_0(self, p): self.print_p(p) p[0] = p[1] + p[2] + self.flatten(p[3]) + p[4] - + def p_function_1(self, p): ''' function : IDENTIFIER LEFTPAREN listitems COMMA RIGHTPAREN From 10cf4610401bee3e61d0e31bf9d099c8fe8022cb Mon Sep 17 00:00:00 2001 From: Steve Balbach Date: Wed, 7 Aug 2019 11:54:01 -0500 Subject: [PATCH 09/13] correct function handling; add more tests; make more pythonic --- src/hcl/parser.py | 37 +++++--------- tests/fixtures/function.hcl | 7 +++ tests/fixtures/function.json | 13 +++++ tests/test_decoder.py | 1 + tests/test_load_dump.py | 99 ++++++++++++++++++++++++++++++++++++ 5 files changed, 132 insertions(+), 25 deletions(-) create mode 100644 tests/fixtures/function.hcl create mode 100644 tests/fixtures/function.json create mode 100644 tests/test_load_dump.py diff --git a/src/hcl/parser.py b/src/hcl/parser.py index 4ae6e14..227e6bc 100644 --- a/src/hcl/parser.py +++ b/src/hcl/parser.py @@ -330,7 +330,7 @@ def p_function_0(self, p): self.print_p(p) p[0] = p[1] + p[2] + self.flatten(p[3]) + p[4] - + def p_function_1(self, p): ''' function : IDENTIFIER LEFTPAREN listitems COMMA RIGHTPAREN @@ -343,31 +343,11 @@ def p_function_1(self, p): def flatten(self, value): returnValue = "" if type(value) is dict: - returnValue += "{" - isFirstTime = True - for key in value: - if isFirstTime: - isFirstTime = False - else: - returnValue += "," - returnValue += key + ":" + self.flatten(value[key]) - returnValue += "}" + returnValue = "{" + ",".join(key + ":" + self.flatten(value[key]) for key in value) + "}" elif type(value) is list: - isFirstTime = True - for v in value: - if isFirstTime: - isFirstTime = False - else: - returnValue += "," - returnValue += self.flatten(v) + returnValue = ",".join(self.flatten(v) for v in value) elif type(value) is tuple: - isFirstTime = True - for v in value: - if isFirstTime: - isFirstTime = False - else: - returnValue += "," - returnValue += self.flatten(v) + returnValue = " ".join(self.flatten(v) for v in value) else: returnValue = value return returnValue @@ -400,13 +380,20 @@ def p_listitems_2(self, p): | objectkey COMMA objectkey | objectkey COMMA object | objectkey COMMA list - | objectkey COMMA IDENTIFIER ASTERISK_PERIOD IDENTIFIER ''' if DEBUG: self.print_p(p) p[0] = [p[1], p[3]] def p_listitems_3(self, p): + ''' + listitems : objectkey COMMA IDENTIFIER ASTERISK_PERIOD IDENTIFIER + ''' + if DEBUG: + self.print_p(p) + p[0] = [p[1], p[3] + p[4] + p[5]] + + def p_listitems_4(self, p): ''' listitems : objectkey list ''' diff --git a/tests/fixtures/function.hcl b/tests/fixtures/function.hcl new file mode 100644 index 0000000..c7c1ee5 --- /dev/null +++ b/tests/fixtures/function.hcl @@ -0,0 +1,7 @@ +data "resource" "object" { + vars = { + cluster_1 = join("\n", data.template_file.cluster_1.*.rendered) + cluster_2 = join("\n", data.template_file.cluster_2.*.rendered) + cluster_3 = format("name_%02d", count.index + 1) + } +} \ No newline at end of file diff --git a/tests/fixtures/function.json b/tests/fixtures/function.json new file mode 100644 index 0000000..6ccd8e9 --- /dev/null +++ b/tests/fixtures/function.json @@ -0,0 +1,13 @@ +{ + "data": { + "resource": { + "object": { + "vars": { + "cluster_1": "join(\\n,data.template_file.cluster_1.*.rendered)", + "cluster_2": "join(\\n,data.template_file.cluster_2.*.rendered)", + "cluster_3": "format(name_%02d,count.index + 1)" + } + } + } + } +} \ No newline at end of file diff --git a/tests/test_decoder.py b/tests/test_decoder.py index 60fc071..d4ba6ce 100644 --- a/tests/test_decoder.py +++ b/tests/test_decoder.py @@ -23,6 +23,7 @@ ('flat.hcl', None, {'foo': 'bar', 'Key': 7}), ('float.hcl', None, {'a': 1.02}), ('float.hcl', 'float.json', None), + ('function.hcl', 'function.json', None), ('multiline_bad.hcl', 'multiline.json', None), ('scientific.hcl', 'scientific.json', None), ('structure.hcl', 'structure_flat.json', None), diff --git a/tests/test_load_dump.py b/tests/test_load_dump.py new file mode 100644 index 0000000..5984541 --- /dev/null +++ b/tests/test_load_dump.py @@ -0,0 +1,99 @@ +# +# These tests are taken from hcl/parse_test.go +# + +from __future__ import print_function + +from os.path import join, dirname +import hcl +import json + +import pytest + +PARSE_FIXTURE_DIR = join(dirname(__file__), 'lex-fixtures') +PARSE_FIXTURES = [ + ( + "assign_colon.hcl", + False, + ), + ( + "comment.hcl", + False, + ), + ( + "list_comma.hcl", + False, + ), + ( + "list_of_maps.hcl", + False, + ), + ( + "multiple.hcl", + False, + ), + ( + "structure.hcl", + False, + ), + ( + "structure_basic.hcl", + False, + ), + ( + "structure_empty.hcl", + False, + ), + ( + "complex.hcl", + False, + ), + ( + "assign_deep.hcl", + False, + ), + ( + "types.hcl", + False, + ), + ( + "structure_comma.hcl", + False, + ), + ( + "terraform0.12syntax.hcl", + False, + ), + ( + "conditional_operator.hcl", + False, + ), +] + +@pytest.mark.parametrize("hcl_fname,invalid", PARSE_FIXTURES) +def test_parser_bytes(hcl_fname, invalid): + + with open(join(PARSE_FIXTURE_DIR, hcl_fname), 'rb') as fp: + + input = fp.read() + print(input) + + if not invalid: + hcl.loads(input) + else: + with pytest.raises(ValueError): + hcl.loads(input) + +@pytest.mark.parametrize("hcl_fname,invalid", PARSE_FIXTURES) +def test_parser_str(hcl_fname, invalid): + + with open(join(PARSE_FIXTURE_DIR, hcl_fname), 'r') as fp: + + input = fp.read() + print(input) + + if not invalid: + hcl.loads(input) + else: + with pytest.raises(ValueError): + hcl.loads(input) From c09016105f8e5ce1d0b21e156072ed57ebfdc3e4 Mon Sep 17 00:00:00 2001 From: Steve Balbach Date: Wed, 7 Aug 2019 12:12:34 -0500 Subject: [PATCH 10/13] update for black formatting --- src/hcl/parser.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/hcl/parser.py b/src/hcl/parser.py index 227e6bc..d2e6f88 100644 --- a/src/hcl/parser.py +++ b/src/hcl/parser.py @@ -330,7 +330,7 @@ def p_function_0(self, p): self.print_p(p) p[0] = p[1] + p[2] + self.flatten(p[3]) + p[4] - + def p_function_1(self, p): ''' function : IDENTIFIER LEFTPAREN listitems COMMA RIGHTPAREN @@ -343,7 +343,11 @@ def p_function_1(self, p): def flatten(self, value): returnValue = "" if type(value) is dict: - returnValue = "{" + ",".join(key + ":" + self.flatten(value[key]) for key in value) + "}" + returnValue = ( + "{" + + ",".join(key + ":" + self.flatten(value[key]) for key in value) + + "}" + ) elif type(value) is list: returnValue = ",".join(self.flatten(v) for v in value) elif type(value) is tuple: From 7a5baa2f0c584ada4a95b449410859d00da013fe Mon Sep 17 00:00:00 2001 From: Steve Balbach Date: Wed, 7 Aug 2019 14:44:45 -0500 Subject: [PATCH 11/13] more fixes for function syntax --- src/hcl/parser.py | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/src/hcl/parser.py b/src/hcl/parser.py index d2e6f88..4626ea9 100644 --- a/src/hcl/parser.py +++ b/src/hcl/parser.py @@ -322,24 +322,60 @@ def p_list_2(self, p): self.print_p(p) p[0] = [p[3]] + [p[5] + p[6] + p[7]] + def p_list_of_lists_0(self, p): + ''' + list_of_lists : list COMMA list + ''' + if DEBUG: + self.print_p(p) + p[0] = p[1], p[3] + + def p_list_of_lists_1(self, p): + ''' + list_of_lists : list_of_lists COMMA list + ''' + if DEBUG: + self.print_p(p) + p[0] = p[1] + (p[3],) + def p_function_0(self, p): ''' function : IDENTIFIER LEFTPAREN listitems RIGHTPAREN + | IDENTIFIER LEFTPAREN list_of_lists RIGHTPAREN ''' if DEBUG: self.print_p(p) p[0] = p[1] + p[2] + self.flatten(p[3]) + p[4] - + def p_function_1(self, p): ''' function : IDENTIFIER LEFTPAREN listitems COMMA RIGHTPAREN + | IDENTIFIER LEFTPAREN list_of_lists COMMA RIGHTPAREN ''' if DEBUG: self.print_p(p) p[0] = p[1] + p[2] + self.flatten(p[3]) + p[5] + def p_function_2(self, p): + ''' + function : IDENTIFIER LEFTPAREN list PERIOD PERIOD PERIOD RIGHTPAREN + ''' + if DEBUG: + self.print_p(p) + + p[0] = p[1] + p[2] + self.flatten(p[3]) + p[4] + p[5] + p[6] + p[7] + + def p_function_3(self, p): + ''' + function : IDENTIFIER LEFTPAREN LEFTBRACKET list_of_lists RIGHTBRACKET PERIOD PERIOD PERIOD RIGHTPAREN + ''' + if DEBUG: + self.print_p(p) + + p[0] = p[1] + p[2] + p[3] + self.flatten(p[4]) + p[5] + p[6] + p[7] + p[8] + p[9] + def flatten(self, value): returnValue = "" if type(value) is dict: From d252b08080784cf91f225bb78447396c20e8e4eb Mon Sep 17 00:00:00 2001 From: Steve Balbach Date: Fri, 16 Aug 2019 14:23:50 -0500 Subject: [PATCH 12/13] fix formatting for black --- src/hcl/parser.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/hcl/parser.py b/src/hcl/parser.py index 4626ea9..02b1891 100644 --- a/src/hcl/parser.py +++ b/src/hcl/parser.py @@ -347,7 +347,7 @@ def p_function_0(self, p): self.print_p(p) p[0] = p[1] + p[2] + self.flatten(p[3]) + p[4] - + def p_function_1(self, p): ''' function : IDENTIFIER LEFTPAREN listitems COMMA RIGHTPAREN @@ -366,7 +366,7 @@ def p_function_2(self, p): self.print_p(p) p[0] = p[1] + p[2] + self.flatten(p[3]) + p[4] + p[5] + p[6] + p[7] - + def p_function_3(self, p): ''' function : IDENTIFIER LEFTPAREN LEFTBRACKET list_of_lists RIGHTBRACKET PERIOD PERIOD PERIOD RIGHTPAREN @@ -374,7 +374,9 @@ def p_function_3(self, p): if DEBUG: self.print_p(p) - p[0] = p[1] + p[2] + p[3] + self.flatten(p[4]) + p[5] + p[6] + p[7] + p[8] + p[9] + p[0] = ( + p[1] + p[2] + p[3] + self.flatten(p[4]) + p[5] + p[6] + p[7] + p[8] + p[9] + ) def flatten(self, value): returnValue = "" From 8c81b4a88d6fe9ec10f5b2edc700f92fe6ee5b46 Mon Sep 17 00:00:00 2001 From: Steve Balbach Date: Mon, 6 Jan 2020 11:42:25 -0600 Subject: [PATCH 13/13] update tests --- README.rst | 5 +++++ tests/fixtures/function.hcl | 4 ++-- tests/fixtures/function.json | 6 +++--- tests/test_decoder.py | 1 - 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index de17f6b..f9a49a0 100644 --- a/README.rst +++ b/README.rst @@ -12,6 +12,11 @@ The grammar and many of the tests/fixtures were copied/ported from the golang parser into pyhcl. All releases are tested with a variety of python versions from Python 2.7 onward. +This version has been modified to work with terraform 0.12 syntax. +It should be backward compatible with earlier versions. +It doesn't cover every situation. See discussion in pull request: +https://github.com/virtuald/pyhcl/pull/57 + Installation ============ diff --git a/tests/fixtures/function.hcl b/tests/fixtures/function.hcl index c7c1ee5..e0b155b 100644 --- a/tests/fixtures/function.hcl +++ b/tests/fixtures/function.hcl @@ -1,7 +1,7 @@ data "resource" "object" { vars = { cluster_1 = join("\n", data.template_file.cluster_1.*.rendered) - cluster_2 = join("\n", data.template_file.cluster_2.*.rendered) - cluster_3 = format("name_%02d", count.index + 1) + cluster_2 = format("name_%02d", count.index + 1) + PROXY_AUTH = join(":", [var.proxy_username, var.proxy_password]) } } \ No newline at end of file diff --git a/tests/fixtures/function.json b/tests/fixtures/function.json index 6ccd8e9..c8fb17c 100644 --- a/tests/fixtures/function.json +++ b/tests/fixtures/function.json @@ -3,9 +3,9 @@ "resource": { "object": { "vars": { - "cluster_1": "join(\\n,data.template_file.cluster_1.*.rendered)", - "cluster_2": "join(\\n,data.template_file.cluster_2.*.rendered)", - "cluster_3": "format(name_%02d,count.index + 1)" + "cluster_1": "join('\\n',data.template_file.cluster_1.*.rendered)", + "cluster_2": "format(name_%02d,count.index + 1)", + "PROXY_AUTH": "join(':',var.proxy_username,var.proxy_password)" } } } diff --git a/tests/test_decoder.py b/tests/test_decoder.py index d4ba6ce..60fc071 100644 --- a/tests/test_decoder.py +++ b/tests/test_decoder.py @@ -23,7 +23,6 @@ ('flat.hcl', None, {'foo': 'bar', 'Key': 7}), ('float.hcl', None, {'a': 1.02}), ('float.hcl', 'float.json', None), - ('function.hcl', 'function.json', None), ('multiline_bad.hcl', 'multiline.json', None), ('scientific.hcl', 'scientific.json', None), ('structure.hcl', 'structure_flat.json', None),