diff --git a/RelationalAI.Test/ConvertValueTest.cs b/RelationalAI.Test/ConvertValueTest.cs new file mode 100644 index 0000000..7e14c5a --- /dev/null +++ b/RelationalAI.Test/ConvertValueTest.cs @@ -0,0 +1,463 @@ +/* + * Copyright 2022 RelationalAI, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using RelationalAI.Results.Models; +using Xunit; + +namespace RelationalAI.Test +{ + class TestInput + { + public TestInput(string relType, string type, string query, object[] arrowValues, object[] values, bool skip, int? places = null) + { + RelType = relType; + Type = type; + Query = query; + ArrowValues = arrowValues.ToList(); + Values = values.ToList(); + Skip = skip; + Places = places; + } + + public string RelType { get; set; } + public string Type { get; set; } + public string Query { get; set; } + public List ArrowValues { get; set; } + public List Values { get; set; } + public bool Skip { get; set; } + public int? Places { get; set; } + } + + public class ConvertValueTest : UnitTest + { + [Fact] + public void ConvertValueTests() + { + foreach (var test in testInputs) + { + if (!test.Skip) + { + var typeDef = new TypeDef(test.Type); + if (test.Places != null) + { + typeDef.Places = test.Places; + } + + for (int i = 0; i < test.ArrowValues.Count; i++) + { + var value = test.ArrowValues[i]; + var convertedValue = Results.Utils.ConvertValue(typeDef, value); + Assert.Equal(test.Values[i], convertedValue); + } + } + } + } + + readonly TestInput[] testInputs = + { + new TestInput + ( + "String", + "String", + "def output = \"test\"", + new object[]{"test"}, + new object[]{"test"}, + false + ), + new TestInput + ( + "Bool", + "Bool", + "def output = boolean_true, boolean_false", + new object[]{true, false}, + new object[]{true, false}, + false + ), + // FixMe: fails for utf-8 chars + // check using 128077 + new TestInput + ( + "Char", + "Char", + "def output = 'a', 'z'", + new object[]{97, 122}, + new object[]{'a', 'z'}, + false + ), + new TestInput + ( + "Dates.DateTime", + "DateTime", + "def output = 2021-10-12T01:22:31+10:00", + new object[]{63769648951000}, + new object[]{ DateTime.Parse("2021-10-11T15:22:31Z").ToUniversalTime() }, + false + ), + new TestInput + ( + "Dates.Date", + "Date", + "def output = 2021-10-12", + new object[]{738075L}, + new object[]{ DateTime.Parse("2021-10-12T00:00:00Z").ToUniversalTime() }, + false + ), + new TestInput + ( + "Dates.Year", + "Year", + "def output = Year[2022]", + new object[]{2022L}, + new object[]{2022L}, + false + ), + new TestInput + ( + "Dates.Month", + "Month", + "def output = Month[1]", + new object[]{1}, + new object[]{DateTimeFormatInfo.CurrentInfo.GetAbbreviatedMonthName(1)}, + false + ), + new TestInput + ( + "Dates.Week", + "Week", + "def output = Week[1]", + new object[]{1}, + new object[]{1}, + false + ), + new TestInput + ( + "Dates.Day", + "Day", + "def output = Day[1]", + new object[]{1}, + new object[]{1}, + false + ), + new TestInput + ( + "Dates.Hour", + "Hour", + "def output = Hour[1]", + new object[]{1}, + new object[]{1}, + false + ), + new TestInput + ( + "Dates.Minute", + "Minute", + "def output = Minute[1]", + new object[]{1}, + new object[]{1}, + false + ), + new TestInput + ( + "Dates.Second", + "Second", + "def output = Second[1]", + new object[]{1}, + new object[]{1}, + false + ), + new TestInput + ( + "Dates.Millisecond", + "Millisecond", + "def output = Millisecond[1]", + new object[]{1}, + new object[]{1}, + false + ), + new TestInput + ( + "Dates.Microsecond", + "Microsecond", + "def output = Microsecond[1]", + new object[]{1}, + new object[]{1}, + false + ), + new TestInput + ( + "Dates.Nanosecond", + "Nanosecond", + "def output = Nanosecond[1]", + new object[]{1}, + new object[]{1}, + false + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/25 + new TestInput + ( + "HashValue", + "Hash", + @" + entity type Foo = Int + def output = ^Foo[12] + ", + new object[]{1}, + new object[]{1}, + true + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/25 + new TestInput + ( + "Missing", + "Missing", + @"def output = missing", + new object[]{1}, + new object[]{1}, + true + ), + new TestInput + ( + "FilePos", + "FilePos", + @" + def config:data = """""" + a,b,c + 1,2,3 + 4,5,6 + """""" + + def csv = load_csv[config] + + def output(p) = csv(_, p, _) + ", + new object[]{2}, + new object[]{2}, + false + ), + new TestInput + ( + "Int8", + "Int8", + @"def output = int[8, 12], int[8, -12]", + new object[]{Convert.ToSByte(12), Convert.ToSByte(-12)}, + new object[]{Convert.ToSByte(12), Convert.ToSByte(-12)}, + false + ), + new TestInput + ( + "Int16", + "Int16", + @"def output = int[16, 12], int[16, -12]", + new object[]{Convert.ToInt16(12), Convert.ToInt16(-12)}, + new object[]{Convert.ToInt16(12), Convert.ToInt16(-12)}, + false + ), + new TestInput + ( + "Int32", + "Int32", + @"def output = int[32, 12], int[32, -12]", + new object[]{Convert.ToInt32(12), Convert.ToInt32(-12)}, + new object[]{Convert.ToInt32(12), Convert.ToInt32(-12)}, + false + ), + new TestInput + ( + "Int64", + "Int64", + @"def output = int[64, 12], int[64, -12]", + new object[]{Convert.ToInt64(12), Convert.ToInt64(-12)}, + new object[]{Convert.ToInt64(12), Convert.ToInt64(-12)}, + false + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/25 + new TestInput + ( + "Int128", + "Int128", + @"def output = 123456789101112131415, int[128, 0], int[128, -10^10]", + new object[]{Convert.ToInt64(12), Convert.ToInt64(-12)}, + new object[]{Convert.ToInt64(12), Convert.ToInt64(-12)}, + true + ), + new TestInput + ( + "UInt8", + "UInt8", + @"def output = uint[8, 12]", + new object[]{Convert.ToByte(12)}, + new object[]{Convert.ToByte(12)}, + false + ), + new TestInput + ( + "UInt16", + "UInt16", + @"def output = uint[16, 12]", + new object[]{Convert.ToUInt16(12)}, + new object[]{Convert.ToUInt16(12)}, + false + ), + new TestInput + ( + "UInt32", + "UInt32", + @"def output = uint[32, 12]", + new object[]{Convert.ToInt32(12), Convert.ToInt32(-12)}, + new object[]{Convert.ToInt32(12), Convert.ToInt32(-12)}, + false + ), + new TestInput + ( + "UInt64", + "UInt64", + @"def output = uint[64, 12]", + new object[]{Convert.ToUInt64(12)}, + new object[]{Convert.ToUInt64(12)}, + false + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/25 + new TestInput + ( + "UInt128", + "UInt128", + @"def output = 123456789101112131415, int[128, 0], int[128, -10^10]", + new object[]{Convert.ToInt64(12), Convert.ToInt64(-12)}, + new object[]{Convert.ToInt64(12), Convert.ToInt64(-12)}, + true + ), + //FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/30 + new TestInput + ( + "Float16", + "Float16", + @"def output = float[16, 12], float[16, 42.5]", + new object[]{Convert.ToInt64(12), Convert.ToInt64(-12)}, + new object[]{Convert.ToInt64(12), Convert.ToInt64(-12)}, + true + ), + new TestInput + ( + "Float32", + "Float32", + @"def output = float[32, 12], float[32, 42.5]", + new object[]{12.0, 42.5}, + new object[]{12.0, 42.5}, + false + ), + new TestInput + ( + "FixedPointDecimals.FixedDecimal{Int16, 2}", + "Decimal16", + @"def output = parse_decimal[16, 2, ""12.34""]", + new object[]{1234}, + new object[]{12.34m}, + false, + 2 + ), + new TestInput + ( + "FixedPointDecimals.FixedDecimal{Int32, 2}", + "Decimal32", + @"def output = parse_decimal[32, 2, ""12.34""]", + new object[]{1234}, + new object[]{12.34m}, + false, + 2 + ), + new TestInput + ( + "FixedPointDecimals.FixedDecimal{Int64, 2}", + "Decimal64", + @"def output = parse_decimal[64, 2, ""12.34""]", + new object[]{1234}, + new object[]{12.34m}, + false, + 2 + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/25 + new TestInput + ( + "FixedPointDecimals.FixedDecimal{Int128, 2}", + "Decimal28", + @"def output = parse_decimal[128, 2, ""12.34""]", + new object[]{1234}, + new object[]{12.34m}, + true, + 2 + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/25 + new TestInput + ( + "Rational{Int8}", + "Rational8", + @"def output = rational[8, 1, 2]", + new object[]{1234}, + new object[]{12.34m}, + true, + 2 + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/25 + new TestInput + ( + "Rational{Int16}", + "Rational16", + @"def output = rational[16, 1, 2]", + new object[]{1234}, + new object[]{12.34m}, + true, + 2 + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/25 + new TestInput + ( + "Rational{Int32}", + "Rational32", + @"def output = rational[32, 1, 2]", + new object[]{1234}, + new object[]{12.34m}, + true, + 2 + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/25 + new TestInput + ( + "Rational{Int64}", + "Rational64", + @"def output = rational[64, 1, 2]", + new object[]{1234}, + new object[]{12.34m}, + true, + 2 + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/25 + new TestInput + ( + "Rational{Int128}", + "Rational128", + @"def output = rational[128, 1, 2]", + new object[]{1234}, + new object[]{12.34m}, + true, + 2 + ), + }; + } +} diff --git a/RelationalAI.Test/ExecuteAsync.cs b/RelationalAI.Test/ExecuteAsync.cs index edb47c2..621d484 100644 --- a/RelationalAI.Test/ExecuteAsync.cs +++ b/RelationalAI.Test/ExecuteAsync.cs @@ -5,6 +5,8 @@ using Xunit; using System.Threading.Tasks; using RelationalAI.Models.Transaction; +using Apache.Arrow; +using Apache.Arrow.Types; namespace RelationalAI.Test { @@ -13,6 +15,7 @@ public class ExecuteAsyncTests : UnitTest public static string Uuid = Guid.NewGuid().ToString(); public static string Dbname = $"csharp-sdk-{Uuid}"; public static string EngineName = $"csharp-sdk-{Uuid}"; + [Fact] public async Task ExecuteAsyncTest() { @@ -24,21 +27,47 @@ public async Task ExecuteAsyncTest() var query = "x, x^2, x^3, x^4 from x in {1; 2; 3; 4; 5}"; var rsp = await client.ExecuteWaitAsync(Dbname, EngineName, query, true); - var results = new List + // mock arrow table + Schema.Builder builder = new Schema.Builder(); + builder.Field(new Field("v1", Int64Type.Default, false)); + builder.Field(new Field("v2", Int64Type.Default, false)); + builder.Field(new Field("v3", Int64Type.Default, false)); + builder.Field(new Field("v4", Int64Type.Default, false)); + + var recordBatch = new RecordBatch + ( + builder.Build(), + new List + { + new Int64Array.Builder().AppendRange(new List { 1, 2, 3, 4, 5 }).Build(), + new Int64Array.Builder().AppendRange(new List { 1, 4, 9, 16, 25 }).Build(), + new Int64Array.Builder().AppendRange(new List { 1, 8, 27, 64, 125 }).Build(), + new Int64Array.Builder().AppendRange(new List { 1, 16, 81, 256, 625 }).Build() + }, + 5 + ); + + var table = Table.TableFromRecordBatches(recordBatch.Schema, new List { recordBatch }); + + for (int i = 0; i < table.ColumnCount; i++) { - new ArrowRelation("/:output/Int64/Int64/Int64/Int64", new List {1L, 2L, 3L, 4L, 5L} ), - new ArrowRelation("/:output/Int64/Int64/Int64/Int64", new List {1L, 4L, 9L, 16L, 25L} ), - new ArrowRelation("/:output/Int64/Int64/Int64/Int64", new List {1L, 8L, 27L, 64L, 125L} ), - new ArrowRelation("/:output/Int64/Int64/Int64/Int64", new List {1L, 16L, 81L, 256L, 625L} ) - }; + for (int j = 0; j < table.Column(i).Data.ArrayCount; j++) + { + for (int k = 0; k < (table.Column(i).Data.Array(j) as Int64Array).Length; k++) + { + var expected = (table.Column(i).Data.Array(j) as Int64Array).GetValue(k).Value; + var actual = (rsp.Results[0].Table.Column(i).Data.Array(j) as Int64Array).GetValue(k).Value; + Assert.Equal(expected, actual); + } + } + } + // mock proto metadata var metadata = MetadataInfo.Parser.ParseFrom(File.ReadAllBytes("../../../metadata.pb")); + Assert.Equal(metadata.Relations[0].RelationId.Arguments, rsp.Results[0].Metadata.Arguments); - var problems = new List(); - - Assert.Equal(results, rsp.Results); - Assert.Equal(metadata.ToString(), rsp.Metadata.ToString()); - Assert.Equal(problems, rsp.Problems); + // mock problems + Assert.Equal(new List(), rsp.Problems); } public override async Task DisposeAsync() diff --git a/RelationalAI.Test/RelationReaderIntegrationTest.cs b/RelationalAI.Test/RelationReaderIntegrationTest.cs new file mode 100644 index 0000000..c7323b6 --- /dev/null +++ b/RelationalAI.Test/RelationReaderIntegrationTest.cs @@ -0,0 +1,1399 @@ +/* + * Copyright 2022 RelationalAI, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Numerics; +using System.Threading.Tasks; +using RelationalAI.Results; +using RelationalAI.Results.Models; +using Newtonsoft.Json; +using Xunit; + +namespace RelationalAI.Test +{ + class Test + { + public Test(string name, string query, List typeDefs, List values, bool skip) + { + Name = name; + Query = query; + TypeDefs = typeDefs; + Values = values; + Skip = skip; + } + + public string Name { get; set; } + public string Query { get; set; } + public List TypeDefs { get; set; } + public List Values { get; set; } + public bool Skip { get; set; } + } + + public class RelationReaderIntegrationTest : UnitTest + { + public static string Uuid = Guid.NewGuid().ToString(); + public static string DBname = $"csharp-sdk-{Uuid}"; + public static string Engine = $"csharp-sdk-{Uuid}"; + + [Fact] + public async Task StandardTypeIntegrationTestsAsync() + { + var client = CreateClient(); + + await client.CreateEngineWaitAsync(Engine); + await client.CreateDatabaseAsync(DBname, Engine); + + foreach (var test in standardTypeTests) + { + if (!test.Skip) + { + Console.WriteLine($"Test: {test.Name}"); + + var rsp = await client.ExecuteWaitAsync(DBname, Engine, test.Query, true); + Assert.NotNull(rsp); + + var reader = new RelationReader(rsp.Results[0]); + reader.Print(); + var typeDefs = reader.TypeDefs(); + var tuple = reader.Tuple(0); + + Assert.Equal(test.TypeDefs, typeDefs); + Assert.Equal(test.Values, tuple); + } + } + } + + [Fact] + public async Task SpecializationTypeIntegrationTestsAsync() + { + var client = CreateClient(); + + await client.CreateEngineWaitAsync(Engine); + await client.CreateDatabaseAsync(DBname, Engine); + + foreach (var test in specializationTypeTests) + { + if (!test.Skip) + { + Console.WriteLine($"Test: {test.Name}"); + + var rsp = await client.ExecuteWaitAsync(DBname, Engine, test.Query, true); + Assert.NotNull(rsp); + + var reader = new RelationReader(rsp.Results[0]); + reader.Print(); + var typeDefs = reader.TypeDefs(); + var tuple = reader.Tuple(0); + + Assert.Equal(test.TypeDefs, typeDefs); + Assert.Equal(test.Values, tuple); + } + } + } + + [Fact] + public async Task MiscValueTypeIntegrationTestsAsync() + { + var client = CreateClient(); + + await client.CreateEngineWaitAsync(Engine); + await client.CreateDatabaseAsync(DBname, Engine); + + foreach (var test in miscValueTypeTests) + { + if (!test.Skip) + { + Console.WriteLine($"Test: {test.Name}"); + + var rsp = await client.ExecuteWaitAsync(DBname, Engine, test.Query, true); + Assert.NotNull(rsp); + + var reader = new RelationReader(rsp.Results[0]); + reader.Print(); + var typeDefs = reader.TypeDefs(); + var tuple = reader.Tuple(0); + + Assert.Equal(JsonConvert.SerializeObject(test.TypeDefs), JsonConvert.SerializeObject(typeDefs)); + Assert.Equal(test.Values, tuple); + } + } + } + + public override async Task DisposeAsync() + { + var client = CreateClient(); + + try { await client.DeleteDatabaseAsync(DBname); } catch { } + try { await client.DeleteEngineWaitAsync(Engine); } catch { } + } + + readonly List standardTypeTests = new List + { + new Test + ( + "String", + @"def output = ""test""", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("String") + }, + new List {"output", "test" }, + false + ), + new Test + ( + "Bool", + @"def output = boolean_true, boolean_false", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Bool"), + new TypeDef("Bool"), + }, + new List {"output", true, false }, + false + ), + new Test + ( + "Char", + @"def output = 'a', '?'", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Char"), + new TypeDef("Char"), + }, + new List {"output", 'a', '?'}, + false + ), + new Test + ( + "DateTime", + @"def output = 2021-10-12T01:22:31+10:00", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("DateTime"), + }, + new List {"output", DateTime.Parse("2021-10-11T15:22:31Z").ToUniversalTime()}, + false + ), + new Test + ( + "Date", + @"def output = 2021-10-12", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Date"), + }, + new List {"output", DateTime.Parse("2021-10-12Z").ToUniversalTime()}, + false + ), + new Test + ( + "Year", + @"def output = Year[2022]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Year"), + }, + new List {"output", Convert.ToInt64(2022)}, + false + ), + new Test + ( + "Month", + @"def output = Month[1]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Month"), + }, + new List {"output", DateTimeFormatInfo.CurrentInfo.GetAbbreviatedMonthName(1)}, + false + ), + new Test + ( + "Week", + @"def output = Week[1]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Week"), + }, + new List {"output", Convert.ToInt64(1)}, + false + ), + new Test + ( + "Day", + @"def output = Day[1]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Day"), + }, + new List {"output", Convert.ToInt64(1)}, + false + ), + new Test + ( + "Hour", + @"def output = Hour[1]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Hour"), + }, + new List {"output", Convert.ToInt64(1)}, + false + ), + new Test + ( + "Minute", + @"def output = Minute[1]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Minute"), + }, + new List {"output", Convert.ToInt64(1)}, + false + ), + new Test + ( + "Second", + @"def output = Second[1]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Second"), + }, + new List {"output", Convert.ToInt64(1)}, + false + ), + new Test + ( + "Millisecond", + @"def output = Millisecond[1]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Millisecond"), + }, + new List {"output", Convert.ToInt64(1)}, + false + ), + new Test + ( + "Microsecond", + @"def output = Microsecond[1]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Microsecond"), + }, + new List {"output", Convert.ToInt64(1)}, + false + ), + new Test + ( + "Nanosecond", + @"def output = Nanosecond[1]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Nanosecond"), + }, + new List {"output", Convert.ToInt64(1)}, + false + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/25 + new Test + ( + "Hash", + @" + entity type Foo = Int + def output = ^Foo[12] + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Hash"), + }, + null, + true + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/25 + new Test + ( + "Missing", + @"def output = missing", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Missing"), + }, + null, + true + ), + new Test + ( + "FilePos", + @" + def config:data = """""" + a,b,c + 1,2,3 + 4,5,6 + """""" + + def csv = load_csv[config] + + def output(p) = csv(_, p, _) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("FilePos"), + }, + new List {"output", Convert.ToInt64(2)}, + false + ), + new Test + ( + "Int8", + @"def output = int[8, 12], int[8, -12]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Int8"), + new TypeDef("Int8"), + }, + new List {"output", Convert.ToSByte(12), Convert.ToSByte(-12)}, + false + ), + new Test + ( + "Int16", + @"def output = int[16, 123], int[16, -123]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Int16"), + new TypeDef("Int16"), + }, + new List {"output", Convert.ToInt16(123), Convert.ToInt16(-123)}, + false + ), + new Test + ( + "Int32", + @"def output = int[32, 1234], int[32, -1234]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Int32"), + new TypeDef("Int32"), + }, + new List {"output", Convert.ToInt32(1234), Convert.ToInt32(-1234)}, + false + ), + new Test + ( + "Int64", + @"def output = int[64, 12345], int[64, -12345]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Int64"), + new TypeDef("Int64"), + }, + new List {"output", Convert.ToInt64(12345), Convert.ToInt64(-12345)}, + false + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/25 + new Test + ( + "Int128", + @"def output = 123456789101112131415, int[128, 0], int[128, -10^10]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Int128"), + new TypeDef("Int128"), + new TypeDef("Int128"), + }, + null, + true + ), + new Test + ( + "UInt8", + @"def output = uint[8, 12]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("UInt8"), + }, + new List {"output", Convert.ToByte(12)}, + false + ), + new Test + ( + "UInt16", + @"def output = uint[16, 123]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("UInt16"), + }, + new List {"output", Convert.ToUInt16(123)}, + false + ), + new Test + ( + "UInt32", + @"def output = uint[32, 1234]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("UInt32"), + }, + new List {"output", Convert.ToUInt32(1234)}, + false + ), + new Test + ( + "UInt64", + @"def output = uint[64, 12345]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("UInt64"), + }, + new List {"output", Convert.ToUInt64(12345)}, + false + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/25 + new Test + ( + "Int128", + @"def output = uint[128, 123456789101112131415], uint[128, 0], 0xdade49b564ec827d92f4fd30f1023a1e", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Int128"), + new TypeDef("Int128"), + new TypeDef("Int128"), + }, + null, + true + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/30 + new Test + ( + "Float16", + @"def output = float[16, 12], float[16, 42.5]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Float16"), + new TypeDef("Float16"), + }, + new List {"output", 12, 42.5}, + true + ), + new Test + ( + "Float32", + @"def output = float[32, 12], float[32, 42.5]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Float32"), + new TypeDef("Float32"), + }, + new List {"output", Convert.ToSingle(12), Convert.ToSingle(42.5)}, + false + ), + new Test + ( + "Float64", + @"def output = float[64, 12], float[64, 42.5]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Float64"), + new TypeDef("Float64"), + }, + new List {"output", Convert.ToDouble(12), Convert.ToDouble(42.5)}, + false + ), + new Test + ( + "Decimal16", + @"def output = parse_decimal[16, 2, ""12.34""]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Decimal16", places: 2), + }, + new List {"output", 12.34m}, + false + ), + new Test + ( + "Decimal32", + @"def output = parse_decimal[32, 2, ""12.34""]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Decimal32", places: 2), + }, + new List {"output", 12.34m}, + false + ), + new Test + ( + "Decimal64", + @"def output = parse_decimal[64, 2, ""12.34""]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Decimal64", places: 2), + }, + new List {"output", 12.34m}, + false + ), + // FIXME + new Test + ( + "Decimal64", + @"def output = parse_decimal[64, 2, ""12345678901011121314.34""]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Decimal64", places: 2), + }, + new List {"output", 12345678901011121314.34m}, + true + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/30 + new Test + ( + "Rational8", + @"def output = rational[8, 1, 2]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Rational8"), + }, + null, + true + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/30 + new Test + ( + "Rational16", + @"def output = rational[16, 1, 2]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Rational16"), + }, + null, + true + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/30 + new Test + ( + "Rational32", + @"def output = rational[32, 1, 2]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Rational32"), + }, + null, + true + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/30 + new Test + ( + "Rational64", + @"def output = rational[64, 1, 2]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Rational64"), + }, + null, + true + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/30 + new Test + ( + "Rational128", + @"def output = rational[128, 123456789101112313, 9123456789101112313]", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Rational128"), + }, + null, + true + ), + }; + + readonly List specializationTypeTests = new List + { + new Test + ( + "String(symbol)", + @"def output = :foo", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("String", "foo")), + }, + new List {"output", "foo"}, + false + ), + new Test + ( + "String", + @" + def v = ""foo"" + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("String", "foo")), + }, + new List {"output", "foo"}, + false + ), + new Test + ( + "String with slash", + @" + def v = ""foo / bar"" + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("String", "foo / bar")), + }, + new List {"output", "foo / bar"}, + false + ), + new Test + ( + "Char", + @" + def v = '?' + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Char", '?')), + }, + new List {"output", '?'}, + false + ), + // FIXME: DateTime precision + new Test + ( + "DateTime", + @" + def v = 2021-10-12T01:22:31+10:00 + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("DateTime", DateTime.Parse("2022-10-11T15:22:31Z").ToUniversalTime())), + }, + new List {"output", DateTime.Parse("2022-10-11T15:22:31Z").ToUniversalTime()}, + false + ), + new Test + ( + "Date", + @" + def v = 2021-10-12 + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Date", DateTime.Parse("2021-10-12Z").ToUniversalTime())), + }, + new List {"output", DateTime.Parse("2021-10-12Z").ToUniversalTime()}, + false + ), + new Test + ( + "Year", + @" + def v = Year[2022] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Year", Convert.ToInt64(2022))), + }, + new List {"output", Convert.ToInt64(2022)}, + false + ), + new Test + ( + "Month", + @" + def v = Month[1] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Month", DateTimeFormatInfo.CurrentInfo.GetAbbreviatedMonthName(1))), + }, + new List {"output", DateTimeFormatInfo.CurrentInfo.GetAbbreviatedMonthName(1)}, + false + ), + new Test + ( + "Week", + @" + def v = Week[1] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Week", Convert.ToInt64(1))), + }, + new List {"output", Convert.ToInt64(1)}, + false + ), + new Test + ( + "Day", + @" + def v = Day[1] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Day", Convert.ToInt64(1))), + }, + new List {"output", Convert.ToInt64(1)}, + false + ), + new Test + ( + "Hour", + @" + def v = Hour[1] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Hour", Convert.ToInt64(1))), + }, + new List {"output", Convert.ToInt64(1)}, + false + ), + new Test + ( + "Minute", + @" + def v = Minute[1] + def output = #(v) + " + , + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Minute", Convert.ToInt64(1))), + }, + new List {"output", Convert.ToInt64(1)}, + false + ), + new Test + ( + "Second", + @" + def v = Second[1] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Second", Convert.ToInt64(1))), + }, + new List {"output", Convert.ToInt64(1)}, + false + ), + new Test + ( + "Millisecond", + @" + def v = Millisecond[1] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Millisecond", Convert.ToInt64(1))), + }, + new List {"output", Convert.ToInt64(1)}, + false + ), + new Test + ( + "Microsecond", + @" + def v = Microsecond[1] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Microsecond", Convert.ToInt64(1))), + }, + new List {"output", Convert.ToInt64(1)}, + false + ), + new Test + ( + "Nanosecond", + @" + def v = Nanosecond[1] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Nanosecond", Convert.ToInt64(1))), + }, + new List {"output", Convert.ToInt64(1)}, + false + ), + new Test + ( + "Hash", + @" + entity type Foo = Int + def v = ^Foo[12] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Hash", BigInteger.Parse("290925887971139297379988470542779955742"))), + }, + new List {"output", BigInteger.Parse("290925887971139297379988470542779955742")}, + false + ), + new Test + ( + "Missing", + @" + def v = missing + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Missing", null)), + }, + new List {"output", null}, + false + ), + new Test + ( + "FilePos", + @" + def config:data = """""" + a,b,c + 1,2,3 + 4,5,6 + """""" + + def csv = load_csv[config] + + def v(p) = csv(_, p, _) + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("FilePos", Convert.ToInt64(2))), + }, + new List {"output", Convert.ToInt64(2)}, + false + ), + new Test + ( + "Int8", + @" + def v = int[8, 12] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Int8", Convert.ToInt32(12))), + }, + new List {"output", Convert.ToInt32(12)}, + false + ), + new Test + ( + "Int16", + @" + def v = int[16, 123] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Int16", Convert.ToInt32(123))), + }, + new List {"output", Convert.ToInt32(123)}, + false + ), + new Test + ( + "Int32", + @" + def v = int[32, 1234] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Int32", Convert.ToInt32(1234))), + }, + new List {"output", Convert.ToInt32(1234)}, + false + ), + new Test + ( + "Int64", + @" + def v = int[64, -12345] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Int64", Convert.ToInt64(-12345))), + }, + new List {"output", Convert.ToInt64(-12345)}, + false + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/25 + new Test + ( + "Int128", + @" + def v = int[128, -123456789101112131415] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Int128", BigInteger.Parse("-123456789101112131415"))), + }, + new List {"output", BigInteger.Parse("-123456789101112131415")}, + false + ), + new Test + ( + "UInt8", + @" + def v = uint[8, 12] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("UInt8", Convert.ToUInt32(12))), + }, + new List {"output", Convert.ToUInt32(12)}, + false + ), + new Test + ( + "UInt16", + @" + def v = uint[16, 123] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("UInt16", Convert.ToUInt32(123))), + }, + new List {"output", Convert.ToUInt32(123)}, + false + ), + new Test + ( + "UInt32", + @" + def v = uint[32, 1234] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("UInt32", Convert.ToUInt32(1234))), + }, + new List {"output", Convert.ToUInt32(1234)}, + false + ), + new Test + ( + "UInt64", + @" + def v = uint[64, 12345] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("UInt64", Convert.ToUInt64(12345))), + }, + new List {"output", Convert.ToUInt64(12345)}, + false + ), + new Test + ( + "UInt128", + @" + def v = uint[128, 123456789101112131415] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("UInt128", BigInteger.Parse("123456789101112131415"))), + }, + new List {"output", BigInteger.Parse("123456789101112131415")}, + false + ), + new Test + ( + "Float16", + @" + def v = float[16, 42.5] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Float16", Convert.ToSingle(42.5))), + }, + new List {"output", Convert.ToSingle(42.5)}, + false + ), + new Test + ( + "Float32", + @" + def v = float[32, 42.5] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Float32", Convert.ToSingle(42.5))), + }, + new List {"output", Convert.ToSingle(42.5)}, + false + ), + new Test + ( + "Float64", + @" + def v = float[64, 42.5] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Float64", Convert.ToDouble(42.5))), + }, + new List {"output", Convert.ToDouble(42.5)}, + false + ), + new Test + ( + "Decimal16", + @" + def v = parse_decimal[16, 2, ""12.34""] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Decimal16", 12.34m, 2)), + }, + new List {"output", 12.34m}, + false + ), + new Test + ( + "Decimal32", + @" + def v = parse_decimal[32, 2, ""12.34""] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Decimal32", 12.34m, 2)), + }, + new List {"output", 12.34m}, + false + ), + new Test + ( + "Decimal64", + @" + def v = parse_decimal[64, 2, ""12.34""] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Decimal64", 12.34m, 2)), + }, + new List {"output", 12.34m}, + false + ), + // FIXME + new Test + ( + "Decimal64", + @" + def v = parse_decimal[64, 2, ""12345678901011121314.34""] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Decimal64", 12345678901011121314.34m, 2)), + }, + new List {"output", 12345678901011121314.34m}, + true + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/30 + new Test + ( + "Rational8", + @" + def v = rational[8, 1, 2] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Rational8", "1 / 2")), + }, + new List {"output", "1 / 2"}, + false + ), + new Test + ( + "Rational16", + @" + def v = rational[16, 1, 2] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Rational16", "1 / 2")), + }, + new List {"output", "1 / 2"}, + false + ), + new Test + ( + "Rational32", + @" + def v = rational[32, 1, 2] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Rational32", "1 / 2")), + }, + new List {"output", "1 / 2"}, + false + ), + new Test + ( + "Rational64", + @" + def v = rational[64, 1, 2] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Rational64", "1 / 2")), + }, + new List {"output", "1 / 2"}, + false + ), + new Test + ( + "Rational128", + @" + def v = rational[128, 123456789101112313, 9123456789101112313] + def output = #(v) + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef("Constant", new TypeDef("Rational128", "123456789101112313 / 9123456789101112313")), + }, + new List {"output", "123456789101112313 / 9123456789101112313"}, + false + ), + }; + + readonly List miscValueTypeTests = new List + { + new Test + ( + "Int", + @" + value type MyType = Int + def output = ^MyType[123] + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef( + "ValueType", + typeDefs: new List + { + new TypeDef("Constant", new TypeDef("String", "MyType")), + new TypeDef("Int64") + } + ), + }, + new List {"output", new object[]{"MyType", Convert.ToInt64(123)}}, + false + ), + // FIXME: https://github.com/RelationalAI/rai-sdk-csharp/issues/30 + new Test + ( + "Int128", + @" + value type MyType = SignedInt[128] + def output = ^MyType[123445677777999999999] + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef( + "ValueType", + typeDefs: new List + { + new TypeDef("Constant", new TypeDef("String", "MyType")), + new TypeDef("Int128") + } + ), + }, + new List {"output", new object[]{"MyType", BigInteger.Parse("123445677777999999999")}}, + true + ), + new Test + ( + "Date", + @" + value type MyType = Date + def output = ^MyType[2021-10-12] + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef( + "ValueType", + typeDefs: new List + { + new TypeDef("Constant", new TypeDef("String", "MyType")), + new TypeDef("Date") + } + ), + }, + new List {"output", new object[]{"MyType", DateTime.Parse("2021-10-12Z").ToUniversalTime()} }, + false + ), + new Test + ( + "OuterType(InnerType(Int, String), String)", + @" + value type InnerType = Int, String + value type OuterType = InnerType, String + def output = ^OuterType[^InnerType[123, ""inner""], ""outer""] + ", + new List + { + new TypeDef("Constant", new TypeDef("String", "output")), + new TypeDef( + "ValueType", + typeDefs: new List + { + new TypeDef("Constant", new TypeDef("String", "OuterType")), + new TypeDef( + "ValueType", + typeDefs: new List + { + new TypeDef("Constant", new TypeDef("String", "InnerType")), + new TypeDef("Int64"), + new TypeDef("String"), + } + ), + new TypeDef("String") + } + ), + }, + new List {"output", new object[] { "OuterType", new object[] {"InnerType", Convert.ToInt64(123), "inner"}, "outer"} }, + false + ), + }; + } +} diff --git a/RelationalAI.Test/RelationReaderTest.cs b/RelationalAI.Test/RelationReaderTest.cs new file mode 100644 index 0000000..5e8c612 --- /dev/null +++ b/RelationalAI.Test/RelationReaderTest.cs @@ -0,0 +1,218 @@ +/* + * Copyright 2022 RelationalAI, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; +using Apache.Arrow; +using Apache.Arrow.Types; +using Google.Protobuf; +using Newtonsoft.Json; +using Relationalai.Protocol; +using RelationalAI.Results; +using RelationalAI.Results.Models; +using Xunit; + +namespace RelationalAI.Test +{ + public class RelationReaderTest : UnitTest + { + // query + // def output = + // (#1, :foo, "w", :bar, 'a', 1); + // (#1, :foo, "x", :bar, 'b', 2); + // (#1, :foo, "y", :bar, 'c', 3); + // (#1, :foo, "z", :bar, 'd', 4) + + [Fact] + public void TypesDefinitionTests() + { + var expected = new List + { + new ColumnDef + ( + new TypeDef("Constant", new TypeDef("String", "output")), + new RelType(), + 0 + ), + new ColumnDef + ( + new TypeDef("Constant", new TypeDef("Int64", 1)), + new RelType(), + 0 + ), + new ColumnDef + ( + new TypeDef("Constant", new TypeDef("String", "foo")), + new RelType(), + 0 + ), + new ColumnDef + ( + new TypeDef("String"), new RelType(), 0 + ), + new ColumnDef + ( + new TypeDef("Constant", new TypeDef("String", "bar")), + new RelType(), + 1 + ), + new ColumnDef + ( + new TypeDef("Char"), new RelType(), 1 + ), + new ColumnDef + ( + new TypeDef("Int64"), new RelType(), 2 + ), + }; + + var reader = MockRelationReader(); + + for (int i = 0; i < reader.ColumnDefs.Count; i++) + { + Assert.Equal(reader.ColumnDefs[i].ArrowIndex, expected[i].ArrowIndex); + Assert.Equal(reader.ColumnDefs[i].TypeDef, expected[i].TypeDef); + } + } + + RelationReader MockRelationReader() + { + var reader = new RelationReader + { + RelationID = MockRelationID() as RelationId, + Table = MockArrowTable() + }; + + reader.ColumnDefs = reader.GetColumnDefsFromProtobuf(); + + return reader; + } + + IMessage MockRelationID() + { + var relationId = JsonParser.Default.Parse(metadataString, RelationId.Descriptor); + return relationId; + } + + Table MockArrowTable() + { + Schema.Builder builder = new Schema.Builder(); + builder.Field(new Field("v1", StringType.Default, false)); + builder.Field(new Field("v2", UInt32Type.Default, false)); + builder.Field(new Field("v3", Int64Type.Default, false)); + + var recordBatch = new RecordBatch + ( + builder.Build(), + new List + { + new StringArray.Builder().AppendRange(new List { "w", "x", "y", "z" }).Build(), + new UInt32Array.Builder().AppendRange(new List { 97, 98, 99, 100 }).Build(), + new Int64Array.Builder().AppendRange(new List { 1, 2, 3, 4 }).Build(), + }, + 4 + ); + + return Table.TableFromRecordBatches(recordBatch.Schema, new List { recordBatch }); + } + + readonly string metadataString = @" + { + ""arguments"": [ + { + ""tag"": ""CONSTANT_TYPE"", + ""constantType"": { + ""relType"": { + ""tag"": ""PRIMITIVE_TYPE"", + ""primitiveType"": ""STRING"" + }, + ""value"": { + ""arguments"": [ + { + ""tag"": ""STRING"", + ""stringVal"": ""b3V0cHV0"" + } + ] + } + } + }, + { + ""tag"": ""CONSTANT_TYPE"", + ""constantType"": { + ""relType"": { + ""tag"": ""PRIMITIVE_TYPE"", + ""primitiveType"": ""INT_64"" + }, + ""value"": { + ""arguments"": [ + { + ""tag"": ""INT_64"", + ""int64Val"": ""1"" + } + ] + } + } + }, + { + ""tag"": ""CONSTANT_TYPE"", + ""constantType"": { + ""relType"": { + ""tag"": ""PRIMITIVE_TYPE"", + ""primitiveType"": ""STRING"" + }, + ""value"": { + ""arguments"": [ + { + ""tag"": ""STRING"", + ""stringVal"": ""Zm9v"" + } + ] + } + } + }, + { + ""tag"": ""PRIMITIVE_TYPE"", + ""primitiveType"": ""STRING"" + }, + { + ""tag"": ""CONSTANT_TYPE"", + ""constantType"": { + ""relType"": { + ""tag"": ""PRIMITIVE_TYPE"", + ""primitiveType"": ""STRING"" + }, + ""value"": { + ""arguments"": [ + { + ""tag"": ""STRING"", + ""stringVal"": ""YmFy"" + } + ] + } + } + }, + { + ""tag"": ""PRIMITIVE_TYPE"", + ""primitiveType"": ""CHAR"" + }, + { + ""tag"": ""PRIMITIVE_TYPE"", + ""primitiveType"": ""INT_64"" + } + ] +} + "; + } +} diff --git a/RelationalAI/Models/Transaction/ArrowRelation.cs b/RelationalAI/Models/Transaction/ArrowRelation.cs index 25e68e1..29a6818 100644 --- a/RelationalAI/Models/Transaction/ArrowRelation.cs +++ b/RelationalAI/Models/Transaction/ArrowRelation.cs @@ -14,28 +14,33 @@ * limitations under the License. */ -using System.Collections.Generic; -using System.Linq; +using Apache.Arrow; +using Relationalai.Protocol; namespace RelationalAI.Models.Transaction { public class ArrowRelation : Entity { - public ArrowRelation(string relationId, List table) + public ArrowRelation(string relationId, Table table, RelationId metadata) { RelationId = relationId; Table = table; + Metadata = metadata; } public string RelationId { get; } - public List Table { get; } + public Table Table { get; } + + public RelationId Metadata { get; } public override bool Equals(object obj) { if (obj is ArrowRelation arrowRelation) { - return RelationId == arrowRelation.RelationId && Table.SequenceEqual(arrowRelation.Table); + return RelationId == arrowRelation.RelationId && + Table == arrowRelation.Table && + Metadata == arrowRelation.Metadata; } return false; diff --git a/RelationalAI/Models/Transaction/ArrowResult.cs b/RelationalAI/Models/Transaction/ArrowResult.cs new file mode 100644 index 0000000..0183447 --- /dev/null +++ b/RelationalAI/Models/Transaction/ArrowResult.cs @@ -0,0 +1,36 @@ +/* + * Copyright 2022 RelationalAI, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Apache.Arrow; + +namespace RelationalAI.Models.Transaction +{ + public class ArrowResult + { + public ArrowResult(string relationID, string filename, Table table) + { + RelationID = relationID; + Filename = filename; + Table = table; + } + + public string RelationID { get; set; } + + public string Filename { get; set; } + + public Table Table { get; set; } + } +} diff --git a/RelationalAI/Models/Transaction/TransactionAsyncResult.cs b/RelationalAI/Models/Transaction/TransactionResponse.cs similarity index 94% rename from RelationalAI/Models/Transaction/TransactionAsyncResult.cs rename to RelationalAI/Models/Transaction/TransactionResponse.cs index e2a2882..eed8a53 100644 --- a/RelationalAI/Models/Transaction/TransactionAsyncResult.cs +++ b/RelationalAI/Models/Transaction/TransactionResponse.cs @@ -19,9 +19,9 @@ namespace RelationalAI.Models.Transaction { - public class TransactionAsyncResult : Entity + public class TransactionResponse : Entity { - public TransactionAsyncResult( + public TransactionResponse( TransactionAsyncCompactResponse transaction, List results, MetadataInfo metadata, diff --git a/RelationalAI/RelationalAI.csproj b/RelationalAI/RelationalAI.csproj index 756944d..7852193 100644 --- a/RelationalAI/RelationalAI.csproj +++ b/RelationalAI/RelationalAI.csproj @@ -28,11 +28,12 @@ + - + all diff --git a/RelationalAI/Results/Models/ColumnDef.cs b/RelationalAI/Results/Models/ColumnDef.cs new file mode 100644 index 0000000..06fe6c7 --- /dev/null +++ b/RelationalAI/Results/Models/ColumnDef.cs @@ -0,0 +1,37 @@ +/* + * Copyright 2022 RelationalAI, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using RelationalAI.Models; +using Relationalai.Protocol; + +namespace RelationalAI.Results.Models +{ + public class ColumnDef : Entity + { + public ColumnDef(TypeDef typeDef, RelType relType, int arrowIndex) + { + TypeDef = typeDef; + RelType = relType; + ArrowIndex = arrowIndex; + } + + public TypeDef TypeDef { get; set; } + + public RelType RelType { get; set; } + + public int ArrowIndex { get; set; } + } +} diff --git a/RelationalAI/Results/Models/TypeDef.cs b/RelationalAI/Results/Models/TypeDef.cs new file mode 100644 index 0000000..5cb11d4 --- /dev/null +++ b/RelationalAI/Results/Models/TypeDef.cs @@ -0,0 +1,67 @@ +/* + * Copyright 2022 RelationalAI, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; +using Newtonsoft.Json; +using RelationalAI.Models; + +namespace RelationalAI.Results.Models +{ + public class TypeDef : Entity + { + public TypeDef(string type, object value = null, int? places = null, List typeDefs = null) + { + Type = type; + Value = value; + Places = places; + TypeDefs = typeDefs; + } + + public string Type { get; set; } + + public object Value { get; set; } + + public int? Places { get; set; } + + public List TypeDefs { get; set; } + + public override bool Equals(object obj) + { + if (obj is TypeDef typeDef) + { + if (JsonConvert.SerializeObject(Value) != JsonConvert.SerializeObject(typeDef.Value)) + { + return false; + } + + return Type == typeDef.Type && + Places == typeDef.Places && + TypeDefs == typeDef.TypeDefs; + } + + return false; + } + + public override int GetHashCode() + { + var hashcode = Type.GetHashCode(); + hashcode ^= Value.GetHashCode(); + hashcode ^= Places.GetHashCode(); + hashcode ^= TypeDefs.GetHashCode(); + return hashcode; + } + } +} diff --git a/RelationalAI/Results/RelationReader.cs b/RelationalAI/Results/RelationReader.cs new file mode 100644 index 0000000..0b81524 --- /dev/null +++ b/RelationalAI/Results/RelationReader.cs @@ -0,0 +1,237 @@ +/* + * Copyright 2022 RelationalAI, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections; +using System.Collections.Generic; +using Apache.Arrow; +using ConsoleTables; +using RelationalAI.Models.Transaction; +using Relationalai.Protocol; +using RelationalAI.Results.Models; + +namespace RelationalAI.Results +{ + public class RelationReader + { + public RelationReader(ArrowRelation relation = null) + { + if (relation != null) + { + Table = relation.Table; + RelationID = relation.Metadata; + ColumnDefs = GetColumnDefsFromProtobuf(); + } + } + + public Table Table { get; set; } + + public RelationId RelationID { get; set; } + + public List ColumnDefs { get; set; } + + public List Tuples() + { + var output = new List(); + + if (Utils.IsFullySpecialized(ColumnDefs)) + { + var spec = new List(); + foreach (var colDef in ColumnDefs) + { + spec.Add(Utils.ConvertValue(colDef.TypeDef, null)); + } + + output.Add(spec); + } + + var rows = RowsRawValues(); + foreach (var row in rows) + { + output.Add(Utils.ArrowRowToValues(row as List, ColumnDefs)); + } + + return output; + } + + public List Tuple(int index) + { + var output = new List(); + + if (Utils.IsFullySpecialized(ColumnDefs)) + { + var spec = new List(); + foreach (var colDef in ColumnDefs) + { + spec.Add(Utils.ConvertValue(colDef.TypeDef, null)); + } + + output.AddRange(spec); + } + + var rows = RowsRawValues(); + if (rows.Count > index) + { + output.AddRange((IEnumerable)Utils.ArrowRowToValues(rows[index] as List, ColumnDefs)); + } + + return output; + } + + public int ColumnCount() + { + return Table.ColumnCount; + } + + public long TupleCount() + { + return Table.RowCount; + } + + public RelationReader Physical() + { + var reader = new RelationReader + { + ColumnDefs = new List(), + Table = Table + }; + + foreach (var colDef in ColumnDefs) + { + if (colDef.TypeDef.Type != "Constant") + { + reader.ColumnDefs.Add(colDef); + } + } + + return reader; + } + + public void Print() + { + var headers = new List(); + foreach (var colDef in ColumnDefs) + { + if (colDef.TypeDef.Type == "Constant") + { + var value = colDef.TypeDef.Value as TypeDef; + headers.Add(value.Type); + } + else + { + headers.Add(colDef.TypeDef.Type); + } + } + + var table = new ConsoleTable(headers.ToArray()); + + foreach (var tuple in Tuples()) + { + table.AddRow((tuple as List).ToArray()); + } + + table.Write(); + } + + public List TypeDefs() + { + var output = new List(); + + foreach (var colDef in ColumnDefs) + { + output.Add(colDef.TypeDef); + } + + return output; + } + + public List GetColumnDefsFromProtobuf() + { + List colDefs = new List(); + var arrowIndex = 0; + + foreach (var relType in RelationID.Arguments) + { + var typeDef = Utils.GetColumnDefFromProtobuf(relType); + var colDef = new ColumnDef(typeDef, relType, arrowIndex); + if (typeDef.Type != "Constant") + { + arrowIndex++; + } + + colDefs.Add(colDef); + } + + return colDefs; + } + + private Dictionary ColumnsToDict() + { + var output = new Dictionary(); + for (int i = 0; i < Table.ColumnCount; i++) + { + var column = Table.Column(i); + var values = ColumnRawValues(column); + + var key = Table.Column(i).Name; + if (output.ContainsKey(key)) + { + output[key].Add(values); + } + else + { + output.Add(key, values); + } + } + + return output; + } + + private IList ColumnRawValues(Column column) + { + var output = new List(); + + for (int j = 0; j < column.Data.ArrayCount; j++) + { + var values = Utils.ArrowArrayToArray(column.Data.Array(j)); + foreach (var v in values) + { + output.Add(v); + } + } + + return output; + } + + private IList RowsRawValues() + { + var output = new List(); + + var dict = ColumnsToDict(); + for (int rowIndex = 0; rowIndex < Table.RowCount; rowIndex++) + { + var arr = new List(); + foreach (var key in dict.Keys) + { + arr.Add(dict[key][rowIndex]); + } + + output.Add(arr); + } + + return output; + } + } +} \ No newline at end of file diff --git a/RelationalAI/Results/Utils.cs b/RelationalAI/Results/Utils.cs new file mode 100644 index 0000000..a4a8116 --- /dev/null +++ b/RelationalAI/Results/Utils.cs @@ -0,0 +1,481 @@ +/* + * Copyright 2022 RelationalAI, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Globalization; +using System.Numerics; +using System.Text; +using Apache.Arrow; +using Google.Protobuf.Collections; +using Relationalai.Protocol; +using RelationalAI.Results.Models; + +namespace RelationalAI.Results +{ + public static class Utils + { + private const double UnixEpoch = 62135683200000; + private const double MillisPerDay = 24 * 60 * 60 * 1000; + + public static IList ArrowRowToValues(IList row, List colDefs) + { + var output = new List(); + + foreach (var colDef in colDefs) + { + if (colDef.TypeDef.Type == "Constant") + { + output.Add(ConvertValue(colDef.TypeDef, null)); + } + else + { + var value = ConvertValue(colDef.TypeDef, row[colDef.ArrowIndex]); + output.Add(value); + } + } + + return output; + } + + public static List UnflattenConstantValue(TypeDef typeDef, RepeatedField value) + { + return null; + } + + public static TypeDef GetColumnDefFromProtobuf(RelType relType) + { + if (relType.Tag == Kind.ConstantType) + { + var typeDef = GetColumnDefFromProtobuf(relType.ConstantType.RelType); + + if (typeDef.Type != "ValueType") + { + var values = new List(); + foreach (var argument in relType.ConstantType.Value.Arguments) + { + values.Add(MapPrimitiveValue(argument)); + } + + object convertedValue; + if (values.Count == 1) + { + convertedValue = ConvertValue(typeDef, values[0]); + } + else + { + convertedValue = ConvertValue(typeDef, values); + } + + typeDef.Value = convertedValue; + + return new TypeDef("Constant", typeDef); + } + else + { + throw new Exception($"value type not handled"); + } + } + + if (relType.Tag == Kind.PrimitiveType) + { + return relType.PrimitiveType switch + { + PrimitiveType.String => new TypeDef("String"), + PrimitiveType.Char => new TypeDef("Char"), + PrimitiveType.Bool => new TypeDef("Bool"), + PrimitiveType.Int8 => new TypeDef("Int8"), + PrimitiveType.Int16 => new TypeDef("Int16"), + PrimitiveType.Int32 => new TypeDef("Int32"), + PrimitiveType.Int64 => new TypeDef("Int64"), + PrimitiveType.Int128 => new TypeDef("Int128"), + PrimitiveType.Uint8 => new TypeDef("UInt8"), + PrimitiveType.Uint16 => new TypeDef("UInt16"), + PrimitiveType.Uint32 => new TypeDef("UInt32"), + PrimitiveType.Uint64 => new TypeDef("UInt64"), + PrimitiveType.Uint128 => new TypeDef("UInt128"), + PrimitiveType.Float16 => new TypeDef("Float16"), + PrimitiveType.Float32 => new TypeDef("Float32"), + PrimitiveType.Float64 => new TypeDef("Float64"), + _ => throw new Exception($"Unhandled rel primitive type: {relType.PrimitiveType}"), + }; + } + + if (relType.Tag == Kind.ValueType) + { + var typeDefs = new List(); + + foreach (var tp in relType.ValueType.ArgumentTypes) + { + typeDefs.Add(GetColumnDefFromProtobuf(tp)); + } + + var typeDef = new TypeDef("ValueType", typeDefs: typeDefs); + + return MapValueType(typeDef); + } + + return new TypeDef("unknown"); + } + + public static TypeDef MapValueType(TypeDef typeDef) + { + var slice = typeDef.TypeDefs.Count < 3 ? typeDef.TypeDefs.Count : 3; + var relNames = new List(); + + foreach (var tp in typeDef.TypeDefs.GetRange(0, slice)) + { + if (tp.Type == "Constant" && (tp.Value as TypeDef).Type == "String") + { + relNames.Add(tp); + } + } + + if (relNames.Count != 3 || + !((relNames[0].Value as TypeDef).Value as string == "rel" && + (relNames[1].Value as TypeDef).Value as string == "base")) + { + return typeDef; + } + + var standardValueType = (relNames[2].Value as TypeDef).Value as string; + switch (standardValueType) + { + case "DateTime": + case "Date": + case "Year": + case "Month": + case "Week": + case "Day": + case "Hour": + case "Minute": + case "Second": + case "Millisecond": + case "Microsecond": + case "Nanosecond": + case "FilePos": + case "Missing": + case "Hash": + return new TypeDef(standardValueType); + case "FixedDecimal": + if (typeDef.TypeDefs.Count == 6 && + typeDef.TypeDefs[3].Type == "Constant" && + typeDef.TypeDefs[4].Type == "Constant") + { + var bits = (long)(typeDef.TypeDefs[3].Value as TypeDef).Value; + var places = (long)(typeDef.TypeDefs[4].Value as TypeDef).Value; + + if (bits == 16 || bits == 32 || bits == 64 || bits == 128) + { + return new TypeDef($"Decimal{bits}", places: Convert.ToInt32(places)); + } + } + + throw new Exception($"unknow fixed decimal type definition"); + case "Rational": + var typeDefs = typeDef.TypeDefs; + if (typeDefs.Count == 5 || typeDefs.Count == 3) + { + switch (typeDefs[3].Type) + { + case "Int8": + return new TypeDef("Rational8"); + case "Int16": + return new TypeDef("Rational16"); + case "Int32": + return new TypeDef("Rational32"); + case "Int64": + return new TypeDef("Rational64"); + case "Int128": + return new TypeDef("Rational128"); + } + } + + throw new Exception($"unknow rational type definition"); + default: + throw new Exception($"unhandled standard value type: {standardValueType}"); + } + } + + public static object MapPrimitiveValue(PrimitiveValue argument) + { + return argument.ValueCase switch + { + PrimitiveValue.ValueOneofCase.StringVal => Encoding.Default.GetString(argument.StringVal.ToByteArray()), + PrimitiveValue.ValueOneofCase.CharVal => argument.CharVal, + PrimitiveValue.ValueOneofCase.BoolVal => argument.BoolVal, + PrimitiveValue.ValueOneofCase.Int8Val => argument.Int8Val, + PrimitiveValue.ValueOneofCase.Int16Val => argument.Int16Val, + PrimitiveValue.ValueOneofCase.Int32Val => argument.Int32Val, + PrimitiveValue.ValueOneofCase.Int64Val => argument.Int64Val, + PrimitiveValue.ValueOneofCase.Int128Val => new[] { argument.Int128Val.Lowbits, argument.Int128Val.Highbits }, + PrimitiveValue.ValueOneofCase.Uint8Val => argument.Uint8Val, + PrimitiveValue.ValueOneofCase.Uint16Val => argument.Uint16Val, + PrimitiveValue.ValueOneofCase.Uint32Val => argument.Uint32Val, + PrimitiveValue.ValueOneofCase.Uint64Val => argument.Uint64Val, + PrimitiveValue.ValueOneofCase.Uint128Val => new[] { argument.Uint128Val.Lowbits, argument.Uint128Val.Highbits }, + PrimitiveValue.ValueOneofCase.Float16Val => argument.Float16Val, + PrimitiveValue.ValueOneofCase.Float32Val => argument.Float32Val, + PrimitiveValue.ValueOneofCase.Float64Val => argument.Float64Val, + _ => throw new Exception($"unhandled protobuf primitive value map value: {argument.ValueCase}"), + }; + } + + public static object ConvertValue(TypeDef typeDef, object value) + { + switch (typeDef.Type) + { + case "Constant": + return (typeDef.Value as TypeDef).Value; + case "String": + return value; + case "Bool": + return value; + case "Char": + return Convert.ToChar(value); + case "DateTime": + return new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc).AddMilliseconds((long)value - UnixEpoch); + case "Date": + return new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc).AddMilliseconds(((long)value * MillisPerDay) - UnixEpoch); + case "Month": + return DateTimeFormatInfo.CurrentInfo.GetAbbreviatedMonthName(Convert.ToInt32(value)); + case "Year": + case "Day": + case "Week": + case "Hour": + case "Minute": + case "Second": + case "Millisecond": + case "Microsecond": + case "Nanosecond": + case "FilePos": + case "Int8": + case "Int16": + case "Int32": + case "Int64": + case "UInt8": + case "UInt16": + case "UInt32": + case "UInt64": + case "Float16": + case "Float32": + case "Float64": + return value; + case "Int128": + return Int128ToBigInteger(value as ulong[]); + case "UInt128": + return Uint128ToBigInteger(value as ulong[]); + case "Decimal16": + case "Decimal32": + case "Decimal64": + return Convert.ToDecimal(value) / Convert.ToDecimal(Math.Pow(10, (double)typeDef.Places)); + case "Hash": + return Uint128ToBigInteger(value as ulong[]); + case "Missing": + return null; + case "Rational8": + case "Rational16": + case "Rational32": + case "Rational64": + var num = (value as List)[0]; + var denom = (value as List)[1]; + return $"{num} / {denom}"; + case "Rational128": + var numValues = (value as List)[0] as ulong[]; + var denomValues = (value as List)[1] as ulong[]; + return $"{Int128ToBigInteger(numValues)} / {Int128ToBigInteger(denomValues)}"; + case "ValueType": + var physicalIndex = -1; + var values = new List(); + foreach (var tp in typeDef.TypeDefs) + { + if (tp.Type == "Constant") + { + values.Add(ConvertValue(tp, null)); + } + else + { + physicalIndex++; + if (value is IList) + { + values.Add(ConvertValue(tp, (value as List)[physicalIndex])); + } + else + { + values.Add(ConvertValue(tp, value)); + } + } + } + + return values.ToArray(); + default: + throw new Exception($"unhandled convert value type: {typeDef.Type}"); + } + } + + public static IList ArrowArrayToArray(Apache.Arrow.Array array) + { + var output = new List(); + + switch (array.GetType().Name) + { + case "UInt8Array": + for (int i = 0; i < (array as UInt8Array).Length; i++) + { + output.Add((array as UInt8Array).GetValue(i).Value); + } + + break; + case "UInt16Array": + for (int i = 0; i < (array as UInt16Array).Length; i++) + { + output.Add((array as UInt16Array).GetValue(i).Value); + } + + break; + case "UInt32Array": + for (int i = 0; i < (array as UInt32Array).Length; i++) + { + output.Add((array as UInt32Array).GetValue(i).Value); + } + + break; + case "UInt64Array": + for (int i = 0; i < (array as UInt64Array).Length; i++) + { + output.Add((array as UInt64Array).GetValue(i).Value); + } + + break; + case "Int8Array": + for (int i = 0; i < (array as Int8Array).Length; i++) + { + output.Add((array as Int8Array).GetValue(i).Value); + } + + break; + case "Int16Array": + for (int i = 0; i < (array as Int16Array).Length; i++) + { + output.Add((array as Int16Array).GetValue(i).Value); + } + + break; + case "Int32Array": + for (int i = 0; i < (array as Int32Array).Length; i++) + { + output.Add((array as Int32Array).GetValue(i).Value); + } + + break; + case "Int64Array": + for (int i = 0; i < (array as Int64Array).Length; i++) + { + output.Add((array as Int64Array).GetValue(i).Value); + } + + break; + case "FloatArray": + for (int i = 0; i < (array as FloatArray).Length; i++) + { + output.Add((array as FloatArray).GetValue(i).Value); + } + + break; + case "DoubleArray": + for (int i = 0; i < (array as DoubleArray).Length; i++) + { + output.Add((array as DoubleArray).GetValue(i).Value); + } + + break; + case "StringArray": + for (int i = 0; i < (array as StringArray).Length; i++) + { + output.Add((array as StringArray).GetString(i)); + } + + break; + case "BooleanArray": + for (int i = 0; i < (array as BooleanArray).Length; i++) + { + output.Add((array as BooleanArray).GetValue(i).Value); + } + + break; + case "StructArray": + var inner = new List(); + + foreach (var field in (array as StructArray).Fields) + { + inner.AddRange(ArrowArrayToArray(field as Apache.Arrow.Array) as List); + } + + output.Add(inner); + break; + default: + throw new System.Exception($"unhandled arrow array type: {array.GetType().Name}"); + } + + return output; + } + + public static bool IsFullySpecialized(List colDefs) + { + if (colDefs.Count == 0) + { + return false; + } + + foreach (var colDef in colDefs) + { + if (colDef.TypeDef.Type != "Constant") + { + return false; + } + } + + return true; + } + + public static bool IsSpecialized(ColumnDef colDef) + { + if (colDef.TypeDef.Type != "Constant") + { + return false; + } + + return true; + } + + public static BigInteger Uint128ToBigInteger(ulong[] values) + { + var lowBits = values[0]; + var highBits = values[1]; + + return (new BigInteger(BitConverter.GetBytes(highBits), isUnsigned: true) << 64) | lowBits; + } + + public static BigInteger Int128ToBigInteger(ulong[] values) + { + var lowBits = values[0]; + var highBits = values[1]; + + return (new BigInteger(BitConverter.GetBytes(highBits), isUnsigned: false) << 64) | lowBits; + } + } +} diff --git a/RelationalAI/Services/Client.cs b/RelationalAI/Services/Client.cs index a834cf9..2f222bf 100644 --- a/RelationalAI/Services/Client.cs +++ b/RelationalAI/Services/Client.cs @@ -321,7 +321,7 @@ public async Task GetTransactionAsync(string id) return Json.Deserialize(rsp); } - public async Task> GetTransactionResultsAsync(string id) + public async Task> GetTransactionResultsAsync(string id) { var files = await _rest.GetAsync(MakeUrl($"{PathTransactions}/{id}/results")) as List; return _rest.ReadArrowFiles(files); @@ -439,7 +439,7 @@ public async Task ExecuteV1Async( return Json.Deserialize(resp); } - public async Task ExecuteWaitAsync( + public async Task ExecuteWaitAsync( string database, string engine, string source, @@ -462,11 +462,12 @@ public async Task ExecuteWaitAsync( .ExecuteAsync(() => GetTransactionAsync(id)); var transaction = transactionResponse.Transaction; - var results = await GetTransactionResultsAsync(id); + var arrowResults = await GetTransactionResultsAsync(id); var metadata = await GetTransactionMetadataAsync(id); var problems = await GetTransactionProblemsAsync(id); + var results = MakeArrowRelations(arrowResults, metadata); - return new TransactionAsyncResult( + return new TransactionResponse( transaction, results, metadata, @@ -474,7 +475,7 @@ public async Task ExecuteWaitAsync( true); } - public async Task ExecuteAsync( + public async Task ExecuteAsync( string database, string engine, string source, @@ -488,10 +489,10 @@ public async Task ExecuteAsync( if (rsp is string s) { var txn = Json.Deserialize(s); - return new TransactionAsyncResult(txn, new List(), null, new List()); + return new TransactionResponse(txn, new List(), null, new List()); } - return ReadTransactionAsyncResults(rsp as List); + return ReadTransactionResponses(rsp as List); } public Task LoadJsonAsync( @@ -680,7 +681,7 @@ private static List ParseProblemsResult(string rsp) return output; } - private TransactionAsyncResult ReadTransactionAsyncResults(List files) + private TransactionResponse ReadTransactionResponses(List files) { var transaction = files.Find(f => f.Name == "transaction"); var metadata = files.Find(f => f.Name == "metadata.proto"); @@ -705,9 +706,28 @@ private TransactionAsyncResult ReadTransactionAsyncResults(List MakeArrowRelations(List results, MetadataInfo metadata) + { + var output = new List(); + var metadataDict = new Dictionary(); + + foreach (var relation in metadata.Relations) + { + metadataDict.Add(relation.FileName, relation.RelationId); + } + + foreach (ArrowResult result in results) + { + output.Add(new ArrowRelation(result.RelationID, result.Table, metadataDict[result.Filename])); + } + + return output; } private async Task GetResourceAsync(string path, string key = null, Dictionary parameters = null) diff --git a/RelationalAI/Services/Rest.cs b/RelationalAI/Services/Rest.cs index 6d00076..1a76004 100644 --- a/RelationalAI/Services/Rest.cs +++ b/RelationalAI/Services/Rest.cs @@ -128,9 +128,9 @@ public string ReadString(byte[] data) return Encoding.UTF8.GetString(data, 0, data.Length); } - public List ReadArrowFiles(List files) + public List ReadArrowFiles(List files) { - var output = new List(); + var output = new List(); foreach (var file in files) { if ("application/vnd.apache.arrow.stream".Equals(file.ContentType.ToLower())) @@ -141,13 +141,17 @@ public List ReadArrowFiles(List files) }; var reader = new ArrowStreamReader(memoryStream); - RecordBatch recordBatch; - while ((recordBatch = reader.ReadNextRecordBatch()) != null) + RecordBatch record; + List records = new List(); + + while ((record = reader.ReadNextRecordBatch()) != null) { - var df = DataFrame.FromArrowRecordBatch(recordBatch); - output.AddRange(df.Columns.Select(col => col.Cast().ToList()) - .Select(values => new ArrowRelation(file.Name, values))); + records.Add(record); } + + var table = Table.TableFromRecordBatches(records.First().Schema, records); + + output.Add(new ArrowResult(file.Name, file.Filename, table)); } }