From 1bf4c990e0bbef31a7b4d42f42ea5e94b99c7306 Mon Sep 17 00:00:00 2001 From: Tony Lian <1040424979@qq.com> Date: Tue, 6 Nov 2018 12:32:03 -0800 Subject: [PATCH 1/9] Add gather macro --- src/Query.jl | 2 +- src/standalone_query_macros.jl | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Query.jl b/src/Query.jl index d34f0cb2..c1465001 100644 --- a/src/Query.jl +++ b/src/Query.jl @@ -16,7 +16,7 @@ import QueryOperators: Enumerable export @from, @query, @count, Grouping, key export @map, @filter, @groupby, @orderby, @orderby_descending, - @thenby, @thenby_descending, @groupjoin, @join, @mapmany, @take, @drop + @thenby, @thenby_descending, @groupjoin, @join, @mapmany, @take, @drop, @gather include("query_utils.jl") include("query_translation.jl") diff --git a/src/standalone_query_macros.jl b/src/standalone_query_macros.jl index 57034821..ffe7937c 100644 --- a/src/standalone_query_macros.jl +++ b/src/standalone_query_macros.jl @@ -182,6 +182,10 @@ macro map(f) helper_namedtuples_replacement end +macro gather(withIndex = false) + :( i -> QueryOperators.gather(QueryOperators.query(i), $(withIndex))) +end + macro mapmany(source, collectionSelector,resultSelector) collectionSelector_as_anonym_func = helper_replace_anon_func_syntax(collectionSelector) resultSelector_as_anonym_func = helper_replace_anon_func_syntax(resultSelector) From a827fd8b59c7d884fc44a16d985dd42df6f4fd7b Mon Sep 17 00:00:00 2001 From: Tony Lian <1040424979@qq.com> Date: Sat, 17 Nov 2018 21:03:30 -0800 Subject: [PATCH 2/9] Update @gather macro to match the new format --- src/standalone_query_macros.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/standalone_query_macros.jl b/src/standalone_query_macros.jl index ffe7937c..22bb8a70 100644 --- a/src/standalone_query_macros.jl +++ b/src/standalone_query_macros.jl @@ -182,8 +182,8 @@ macro map(f) helper_namedtuples_replacement end -macro gather(withIndex = false) - :( i -> QueryOperators.gather(QueryOperators.query(i), $(withIndex))) +macro gather(args...) + :( i -> QueryOperators.gather(QueryOperators.query(i), $(args...))) end macro mapmany(source, collectionSelector,resultSelector) From f8cc858a501fbcf5c2df338fd0c01e6f47bab0dd Mon Sep 17 00:00:00 2001 From: Tony Lian <1040424979@qq.com> Date: Tue, 20 Nov 2018 21:32:26 -0800 Subject: [PATCH 3/9] Parse `-:Name` syntax --- src/standalone_query_macros.jl | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/standalone_query_macros.jl b/src/standalone_query_macros.jl index 22bb8a70..aa4b87df 100644 --- a/src/standalone_query_macros.jl +++ b/src/standalone_query_macros.jl @@ -183,7 +183,18 @@ macro map(f) end macro gather(args...) - :( i -> QueryOperators.gather(QueryOperators.query(i), $(args...))) + parsedArgs = () + for arg in args + # println("arg ", arg) + # println("type of arg ", typeof(arg)) + if typeof(arg) == Expr + m1 = match(r"^-:(.+)", string(arg)) + parsedArgs = (parsedArgs..., :(QueryOperators.Not($(QuoteNode(Symbol(m1[1])))))) + else + parsedArgs = (parsedArgs..., arg) + end + end + :( i -> QueryOperators.gather(QueryOperators.query(i), $(parsedArgs...))) end macro mapmany(source, collectionSelector,resultSelector) From 0faec873c8a0eae17d07624e42faacd86aa3cbbc Mon Sep 17 00:00:00 2001 From: Tony Lian <1040424979@qq.com> Date: Thu, 29 Nov 2018 18:34:19 -0800 Subject: [PATCH 4/9] Add tests for gather operator --- test/test_standalone.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/test_standalone.jl b/test/test_standalone.jl index fb7beea5..1dfe9584 100644 --- a/test/test_standalone.jl +++ b/test/test_standalone.jl @@ -44,4 +44,12 @@ end @test df2[:c] == ["b","c"] end +@testset "@gather operator" begin + + @test iterate((Year=[2017,2018,2019], US=[1,2,3], EU=[1,2,3], CN=[1,2,3]) |> @gather) == nothing + @test iterate((Year=[2017,2018,2019], US=[1,2,3], EU=[1,2,3], CN=[1,2,3]) |> @gather(:US, :EU, :CN)) == ((key = :US, value = 1, Year = 2017), (Any[(key = :US, value = 1, Year = 2017), (key = :EU, value = 1, Year = 2017), (key = :CN, value = 1, Year = 2017), (key = :US, value = 2, Year = 2018), (key = :EU, value = 2, Year = 2018), (key = :CN, value = 2, Year = 2018), (key = :US, value = 3, Year = 2019), (key = :EU, value = 3, Year = 2019), (key = :CN, value = 3, Year = 2019)], 2)) + @test eltype((Year=[2017,2018,2019], US=[1,2,3], EU=[1,2,3], CN=[1,2,3]) |> @gather(:US, :EU, :CN)) == NamedTuple{(:key, :value, :Year),Tuple{Symbol,Int64,Int64}} + +end + end From 8719d63442d1584d398fe9fc5dd335be477764f6 Mon Sep 17 00:00:00 2001 From: Tony Lian <1040424979@qq.com> Date: Sat, 19 Jan 2019 19:50:31 -0800 Subject: [PATCH 5/9] Add standalone test into runtests --- test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index a7ac1249..8f7d430f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -493,5 +493,5 @@ q = collect(@map(source_df, i->i.children)) include("test_dplyr-syntax.jl") include("test_pipesyntax.jl") - +include("test_standalone.jl") end From 3c1f431a462035e5af4cc43c06148f8587a08ffd Mon Sep 17 00:00:00 2001 From: Tony Lian <1040424979@qq.com> Date: Sat, 19 Jan 2019 19:50:47 -0800 Subject: [PATCH 6/9] Add tests for `@gather` --- test/test_standalone.jl | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/test/test_standalone.jl b/test/test_standalone.jl index 1dfe9584..0b7d32e9 100644 --- a/test/test_standalone.jl +++ b/test/test_standalone.jl @@ -1,4 +1,5 @@ using Query +using QueryOperators using DataFrames using Test @@ -45,11 +46,8 @@ end end @testset "@gather operator" begin - - @test iterate((Year=[2017,2018,2019], US=[1,2,3], EU=[1,2,3], CN=[1,2,3]) |> @gather) == nothing - @test iterate((Year=[2017,2018,2019], US=[1,2,3], EU=[1,2,3], CN=[1,2,3]) |> @gather(:US, :EU, :CN)) == ((key = :US, value = 1, Year = 2017), (Any[(key = :US, value = 1, Year = 2017), (key = :EU, value = 1, Year = 2017), (key = :CN, value = 1, Year = 2017), (key = :US, value = 2, Year = 2018), (key = :EU, value = 2, Year = 2018), (key = :CN, value = 2, Year = 2018), (key = :US, value = 3, Year = 2019), (key = :EU, value = 3, Year = 2019), (key = :CN, value = 3, Year = 2019)], 2)) - @test eltype((Year=[2017,2018,2019], US=[1,2,3], EU=[1,2,3], CN=[1,2,3]) |> @gather(:US, :EU, :CN)) == NamedTuple{(:key, :value, :Year),Tuple{Symbol,Int64,Int64}} - + @test sprint(QueryOperators.show,(Year=[2017,2018,2019], US=[1,2,3], EU=[1,2,3], CN=[1,2,3]) |> @gather(:US, :EU, :CN)) == "9x3 query result\nkey │ value │ Year\n────┼───────┼─────\n:US │ 1 │ 2017\n:EU │ 1 │ 2017\n:CN │ 1 │ 2017\n:US │ 2 │ 2018\n:EU │ 2 │ 2018\n:CN │ 2 │ 2018\n:US │ 3 │ 2019\n:EU │ 3 │ 2019\n:CN │ 3 │ 2019" + @test eltype((Year=[2017,2018,2019], US=[1,2,3], EU=[1,2,3], CN=[1,2,3]) |> @gather(:US, :EU, :CN)) == NamedTuple{(:key, :value, :Year),T} where T<:Tuple end end From 94f24dbf3235a9d94a9e4d63ab615681d4b4ac64 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Tue, 19 Mar 2019 09:45:30 -0700 Subject: [PATCH 7/9] Don't remove @unique --- src/Query.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Query.jl b/src/Query.jl index 77f19c80..d6ed6e02 100644 --- a/src/Query.jl +++ b/src/Query.jl @@ -8,7 +8,7 @@ using QueryOperators export @from, @query, @count, Grouping, key -export @map, @filter, @groupby, @orderby, @orderby_descending, +export @map, @filter, @groupby, @orderby, @orderby_descending, @unique, @thenby, @thenby_descending, @groupjoin, @join, @mapmany, @take, @drop, @gather export @select, @rename, @mutate From f866dbcd775ff811ccca10d6f7cf91ff9b074408 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Tue, 19 Mar 2019 09:45:41 -0700 Subject: [PATCH 8/9] Remove diagnostic statements --- src/standalone_query_macros.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/standalone_query_macros.jl b/src/standalone_query_macros.jl index 0b1247f2..0063b8db 100644 --- a/src/standalone_query_macros.jl +++ b/src/standalone_query_macros.jl @@ -185,8 +185,6 @@ end macro gather(args...) parsedArgs = () for arg in args - # println("arg ", arg) - # println("type of arg ", typeof(arg)) if typeof(arg) == Expr m1 = match(r"^-:(.+)", string(arg)) parsedArgs = (parsedArgs..., :(QueryOperators.Not($(QuoteNode(Symbol(m1[1])))))) From d87b24dabad7d2d3e681cfb5f2fb69ba13cbc341 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Tue, 19 Mar 2019 09:52:49 -0700 Subject: [PATCH 9/9] Fix @gather tests --- test/test_standalone.jl | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test/test_standalone.jl b/test/test_standalone.jl index 7096b991..56711f79 100644 --- a/test/test_standalone.jl +++ b/test/test_standalone.jl @@ -46,8 +46,20 @@ end end @testset "@gather operator" begin - @test sprint(QueryOperators.show,(Year=[2017,2018,2019], US=[1,2,3], EU=[1,2,3], CN=[1,2,3]) |> @gather(:US, :EU, :CN)) == "9x3 query result\nkey │ value │ Year\n────┼───────┼─────\n:US │ 1 │ 2017\n:EU │ 1 │ 2017\n:CN │ 1 │ 2017\n:US │ 2 │ 2018\n:EU │ 2 │ 2018\n:CN │ 2 │ 2018\n:US │ 3 │ 2019\n:EU │ 3 │ 2019\n:CN │ 3 │ 2019" - @test eltype((Year=[2017,2018,2019], US=[1,2,3], EU=[1,2,3], CN=[1,2,3]) |> @gather(:US, :EU, :CN)) == NamedTuple{(:key, :value, :Year),T} where T<:Tuple + source_gather = DataFrame(Year=[2017,2018,2019], US=[1,2,3], EU=[4,5,6], CN=[7,8,9]) + @test source_gather |> @gather(:US, :EU, :CN) |> collect == + [ + (Year = 2017, key = :US, value = 1), + (Year = 2017, key = :EU, value = 4), + (Year = 2017, key = :CN, value = 7), + (Year = 2018, key = :US, value = 2), + (Year = 2018, key = :EU, value = 5), + (Year = 2018, key = :CN, value = 8), + (Year = 2019, key = :US, value = 3), + (Year = 2019, key = :EU, value = 6), + (Year = 2019, key = :CN, value = 9) + ] + @test eltype(source_gather |> @gather(:US, :EU, :CN)) == NamedTuple{(:Year, :key, :value),Tuple{Int, Symbol, Int}} end @testset "@unique operator" begin