Adds PartiqlShapeBuilder with NodeId generation for the StaticType (#485)

* Adds `NodeId` to the `StaticType`

This PR
- adds `NodeId` to `StaticType`; this is to be able to use the `id` as a reference to add additional data to the types out of band.
- makes `AutoNodeIdGenerator` thread-safe
- adds `PartiqlShapeBuilder` and moves some `PartiqlShape` APIs to it; this is to be able to generate unique `NodeId`s for a `PartiqlShape` that includes static types that themselves can include other static types.
- adds a static thread safe `shape_builder` function that provides a convenient way for using `PartiqlShapeBuilder` for creating new shapes.
- prepends existing type macros with `type` such as `type_int!` to make macro names more friendly.
- removes `const` PartiQL types under `partiql-types` in favor of `PartiqlShapeBuilder`.

**_The majority of the diffs are related to the macro renames or replacement with the previous `const` types. The main change is in `partiql-types/src/lib.rs` file._**

Author: am357

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "partiql-conformance-tests/partiql-tests"]
 	path = partiql-conformance-tests/partiql-tests
-	url = git@github.com:partiql/partiql-tests.git
+	url = https://github.com/partiql/partiql-tests.git

CHANGELOG.md

@@ -11,13 +11,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - partiql-ast: improved pretty-printing of `CASE` and various clauses
 - 
 ### Added
+- Added `partiql-common`.
+- Added `NodeId` to `StaticType`.
+- *BREAKING* Added thread-safe `PartiqlShapeBuilder` and automatic `NodeId` generation for the `StaticType`.
+- Added a static thread safe `shape_builder` function that provides a convenient way for using `PartiqlShapeBuilder` for creating new shapes.
 
-### Fixed
+### Removed
+- *BREAKING* Removed `partiql-source-map`.
+- *BREAKING* Removed `const` PartiQL types under `partiql-types` in favor of `PartiqlShapeBuilder`.
+- *BREAKING* Removed `StaticType`'s `new`, `new_non_nullable`, and `as_non-nullable` APIs in favor of `PartiqlShapeBuilder`.
 
 ## [0.10.0]
 ### Changed
 - *BREAKING:* partiql-ast: added modeling of `EXCLUDE`
 - *BREAKING:* partiql-ast: added pretty-printing of `EXCLUDE`
+- *BREAKING* Moved some of the `PartiqlShape` APIs to the `PartiqlShapeBuilder`.
+- *BREAKING* Prepended existing type macros with `type` to make macro names more friendly: e.g., `type_int!`
+- *BREAKING* Moved node id generation and `partiql-source-map` to it.
+- *BREAKING* Changed `AutoNodeIdGenerator` to a thread-safe version
 
 ### Added
 - *BREAKING:* partiql-parser: added parsing of `EXCLUDE`

extension/partiql-extension-ddl/src/ddl.rs

@@ -123,8 +123,8 @@ impl PartiqlBasicDdlEncoder {
             Static::Float64 => out.push_str("DOUBLE"),
             Static::String => out.push_str("VARCHAR"),
             Static::Struct(s) => out.push_str(&self.write_struct(s)?),
-            Static::Bag(b) => out.push_str(&self.write_bag(b)?),
-            Static::Array(a) => out.push_str(&self.write_array(a)?),
+            Static::Bag(b) => out.push_str(&self.write_type_bag(b)?),
+            Static::Array(a) => out.push_str(&self.write_type_array(a)?),
             // non-exhaustive catch-all
             _ => todo!("handle type for {}", ty),
         }
@@ -136,12 +136,18 @@ impl PartiqlBasicDdlEncoder {
         Ok(out)
     }
 
-    fn write_bag(&self, bag: &BagType) -> ShapeDdlEncodeResult<String> {
-        Ok(format!("BAG<{}>", self.write_shape(bag.element_type())?))
+    fn write_type_bag(&self, type_bag: &BagType) -> ShapeDdlEncodeResult<String> {
+        Ok(format!(
+            "type_bag<{}>",
+            self.write_shape(type_bag.element_type())?
+        ))
     }
 
-    fn write_array(&self, arr: &ArrayType) -> ShapeDdlEncodeResult<String> {
-        Ok(format!("ARRAY<{}>", self.write_shape(arr.element_type())?))
+    fn write_type_array(&self, arr: &ArrayType) -> ShapeDdlEncodeResult<String> {
+        Ok(format!(
+            "type_array<{}>",
+            self.write_shape(arr.element_type())?
+        ))
     }
 
     fn write_struct(&self, strct: &StructType) -> ShapeDdlEncodeResult<String> {
@@ -189,8 +195,8 @@ impl PartiqlDdlEncoder for PartiqlBasicDdlEncoder {
         let mut output = String::new();
         let ty = ty.expect_static()?;
 
-        if let Static::Bag(bag) = ty.ty() {
-            let s = bag.element_type().expect_struct()?;
+        if let Static::Bag(type_bag) = ty.ty() {
+            let s = type_bag.element_type().expect_struct()?;
             let mut fields = s.fields().peekable();
             while let Some(field) = fields.next() {
                 output.push_str(&format!("\"{}\" ", field.name()));
@@ -223,41 +229,47 @@ impl PartiqlDdlEncoder for PartiqlBasicDdlEncoder {
 mod tests {
     use super::*;
     use indexmap::IndexSet;
-    use partiql_types::{array, bag, f64, int8, r#struct, str, struct_fields, StructConstraint};
+    use partiql_types::{
+        struct_fields, type_array, type_bag, type_float64, type_int8, type_string, type_struct,
+        PartiqlShapeBuilder, StructConstraint,
+    };
 
     #[test]
     fn ddl_test() {
         let nested_attrs = struct_fields![
             (
                 "a",
-                PartiqlShape::any_of(vec![
-                    PartiqlShape::new(Static::DecimalP(5, 4)),
-                    PartiqlShape::new(Static::Int8),
+                PartiqlShapeBuilder::init_or_get().any_of(vec![
+                    PartiqlShapeBuilder::init_or_get().new_static(Static::DecimalP(5, 4)),
+                    PartiqlShapeBuilder::init_or_get().new_static(Static::Int8),
                 ])
             ),
-            ("b", array![str![]]),
-            ("c", f64!()),
+            ("b", type_array![type_string![]]),
+            ("c", type_float64!()),
         ];
-        let details = r#struct![IndexSet::from([nested_attrs])];
+        let details = type_struct![IndexSet::from([nested_attrs])];
 
         let fields = struct_fields![
-            ("employee_id", int8![]),
-            ("full_name", str![]),
-            ("salary", PartiqlShape::new(Static::DecimalP(8, 2))),
+            ("employee_id", type_int8![]),
+            ("full_name", type_string![]),
+            (
+                "salary",
+                PartiqlShapeBuilder::init_or_get().new_static(Static::DecimalP(8, 2))
+            ),
             ("details", details),
-            ("dependents", array![str![]])
+            ("dependents", type_array![type_string![]])
         ];
-        let ty = bag![r#struct![IndexSet::from([
+        let ty = type_bag![type_struct![IndexSet::from([
             fields,
             StructConstraint::Open(false)
         ])]];
 
-        let expected_compact = r#""employee_id" TINYINT,"full_name" VARCHAR,"salary" DECIMAL(8, 2),"details" STRUCT<"a": UNION<DECIMAL(5, 4),TINYINT>,"b": ARRAY<VARCHAR>,"c": DOUBLE>,"dependents" ARRAY<VARCHAR>"#;
+        let expected_compact = r#""employee_id" TINYINT,"full_name" VARCHAR,"salary" DECIMAL(8, 2),"details" STRUCT<"a": UNION<DECIMAL(5, 4),TINYINT>,"b": type_array<VARCHAR>,"c": DOUBLE>,"dependents" type_array<VARCHAR>"#;
         let expected_pretty = r#""employee_id" TINYINT,
 "full_name" VARCHAR,
 "salary" DECIMAL(8, 2),
-"details" STRUCT<"a": UNION<DECIMAL(5, 4),TINYINT>,"b": ARRAY<VARCHAR>,"c": DOUBLE>,
-"dependents" ARRAY<VARCHAR>"#;
+"details" STRUCT<"a": UNION<DECIMAL(5, 4),TINYINT>,"b": type_array<VARCHAR>,"c": DOUBLE>,
+"dependents" type_array<VARCHAR>"#;
 
         let ddl_compact = PartiqlBasicDdlEncoder::new(DdlFormat::Compact);
         assert_eq!(ddl_compact.ddl(&ty).expect("write shape"), expected_compact);

extension/partiql-extension-ddl/tests/ddl-tests.rs

@@ -1,20 +1,26 @@
 use indexmap::IndexSet;
 use partiql_extension_ddl::ddl::{DdlFormat, PartiqlBasicDdlEncoder, PartiqlDdlEncoder};
-use partiql_types::{bag, int, r#struct, str, struct_fields, StructConstraint, StructField};
-use partiql_types::{BagType, PartiqlShape, Static, StructType};
+use partiql_types::{
+    struct_fields, type_bag, type_int, type_string, type_struct, PartiqlShapeBuilder,
+    StructConstraint, StructField,
+};
+use partiql_types::{BagType, Static, StructType};
 
 #[test]
 fn basic_ddl_test() {
-    let details_fields = struct_fields![("age", int!())];
-    let details = r#struct![IndexSet::from([details_fields])];
+    let details_fields = struct_fields![("age", type_int!())];
+    let details = type_struct![IndexSet::from([details_fields])];
     let fields = [
-        StructField::new("id", int!()),
-        StructField::new("name", str!()),
-        StructField::new("address", PartiqlShape::new_non_nullable(Static::String)),
+        StructField::new("id", type_int!()),
+        StructField::new("name", type_string!()),
+        StructField::new(
+            "address",
+            PartiqlShapeBuilder::init_or_get().new_non_nullable_static(Static::String),
+        ),
         StructField::new_optional("details", details.clone()),
     ]
     .into();
-    let shape = bag![r#struct![IndexSet::from([
+    let shape = type_bag![type_struct![IndexSet::from([
         StructConstraint::Fields(fields),
         StructConstraint::Open(false)
     ])]];

partiql-ast/src/builder.rs

@@ -17,7 +17,8 @@ where
 
     pub fn node<T>(&mut self, node: T) -> AstNode<T> {
         let id = self.id_gen.id();
-        AstNode { id, node }
+        let id = id.read().expect("NodId read lock");
+        AstNode { id: *id, node }
     }
 }
 

partiql-common/src/node.rs

@@ -1,5 +1,6 @@
 use indexmap::IndexMap;
 use std::hash::Hash;
+use std::sync::{Arc, RwLock};
 
 #[cfg(feature = "serde")]
 use serde::{Deserialize, Serialize};
@@ -12,27 +13,37 @@ pub struct NodeId(pub u32);
 
 /// Auto-incrementing [`NodeIdGenerator`]
 pub struct AutoNodeIdGenerator {
-    next_id: NodeId,
+    next_id: Arc<RwLock<NodeId>>,
 }
 
 impl Default for AutoNodeIdGenerator {
     fn default() -> Self {
-        AutoNodeIdGenerator { next_id: NodeId(1) }
+        AutoNodeIdGenerator {
+            next_id: Arc::new(RwLock::from(NodeId(1))),
+        }
     }
 }
 
 /// A provider of 'fresh' [`NodeId`]s.
 pub trait NodeIdGenerator {
+    fn id(&self) -> Arc<RwLock<NodeId>>;
+
     /// Provides a 'fresh' [`NodeId`].
-    fn id(&mut self) -> NodeId;
+    fn next_id(&self) -> NodeId;
 }
 
 impl NodeIdGenerator for AutoNodeIdGenerator {
+    fn id(&self) -> Arc<RwLock<NodeId>> {
+        let id = self.next_id();
+        let mut w = self.next_id.write().expect("NodId write lock");
+        *w = id;
+        Arc::clone(&self.next_id)
+    }
+
     #[inline]
-    fn id(&mut self) -> NodeId {
-        let mut next = NodeId(&self.next_id.0 + 1);
-        std::mem::swap(&mut self.next_id, &mut next);
-        next
+    fn next_id(&self) -> NodeId {
+        let id = &self.next_id.read().expect("NodId read lock");
+        NodeId(id.0 + 1)
     }
 }
 
@@ -41,7 +52,11 @@ impl NodeIdGenerator for AutoNodeIdGenerator {
 pub struct NullIdGenerator {}
 
 impl NodeIdGenerator for NullIdGenerator {
-    fn id(&mut self) -> NodeId {
+    fn id(&self) -> Arc<RwLock<NodeId>> {
+        Arc::new(RwLock::from(self.next_id()))
+    }
+
+    fn next_id(&self) -> NodeId {
         NodeId(0)
     }
 }

partiql-eval/src/eval/eval_expr_wrapper.rs

@@ -4,7 +4,7 @@ use crate::eval::expr::{BindError, EvalExpr};
 use crate::eval::EvalContext;
 use itertools::Itertools;
 
-use partiql_types::{PartiqlShape, Static, TYPE_DYNAMIC};
+use partiql_types::{type_dynamic, PartiqlShape, Static, TYPE_DYNAMIC};
 use partiql_value::Value::{Missing, Null};
 use partiql_value::{Tuple, Value};
 
@@ -413,7 +413,7 @@ impl UnaryValueExpr {
     where
         F: 'static + Fn(&Value) -> Value,
     {
-        Self::create_typed::<STRICT, F>([TYPE_DYNAMIC; 1], args, f)
+        Self::create_typed::<STRICT, F>([type_dynamic!(); 1], args, f)
     }
 
     #[allow(dead_code)]

partiql-eval/src/eval/expr/coll.rs

@@ -4,7 +4,9 @@ use crate::eval::expr::{BindError, BindEvalExpr, EvalExpr};
 
 use itertools::{Itertools, Unique};
 
-use partiql_types::{ArrayType, BagType, PartiqlShape, Static, TYPE_BOOL, TYPE_NUMERIC_TYPES};
+use partiql_types::{
+    type_bool, type_numeric, ArrayType, BagType, PartiqlShape, PartiqlShapeBuilder, Static,
+};
 use partiql_value::Value::{Missing, Null};
 use partiql_value::{BinaryAnd, BinaryOr, Value, ValueIter};
 
@@ -49,21 +51,23 @@ impl BindEvalExpr for EvalCollFn {
                 value.sequence_iter().map_or(Missing, &f)
             })
         }
-        let boolean_elems = [PartiqlShape::any_of([
-            PartiqlShape::new(Static::Array(ArrayType::new(Box::new(TYPE_BOOL)))),
-            PartiqlShape::new(Static::Bag(BagType::new(Box::new(TYPE_BOOL)))),
+        let boolean_elems = [PartiqlShapeBuilder::init_or_get().any_of([
+            PartiqlShapeBuilder::init_or_get()
+                .new_static(Static::Array(ArrayType::new(Box::new(type_bool!())))),
+            PartiqlShapeBuilder::init_or_get()
+                .new_static(Static::Bag(BagType::new(Box::new(type_bool!())))),
         ])];
-        let numeric_elems = [PartiqlShape::any_of([
-            PartiqlShape::new(Static::Array(ArrayType::new(Box::new(
-                PartiqlShape::any_of(TYPE_NUMERIC_TYPES),
+        let numeric_elems = [PartiqlShapeBuilder::init_or_get().any_of([
+            PartiqlShapeBuilder::init_or_get().new_static(Static::Array(ArrayType::new(Box::new(
+                PartiqlShapeBuilder::init_or_get().any_of(type_numeric!()),
+            )))),
+            PartiqlShapeBuilder::init_or_get().new_static(Static::Bag(BagType::new(Box::new(
+                PartiqlShapeBuilder::init_or_get().any_of(type_numeric!()),
             )))),
-            PartiqlShape::new(Static::Bag(BagType::new(Box::new(PartiqlShape::any_of(
-                TYPE_NUMERIC_TYPES,
-            ))))),
         ])];
-        let any_elems = [PartiqlShape::any_of([
-            PartiqlShape::new(Static::Array(ArrayType::new_any())),
-            PartiqlShape::new(Static::Bag(BagType::new_any())),
+        let any_elems = [PartiqlShapeBuilder::init_or_get().any_of([
+            PartiqlShapeBuilder::init_or_get().new_static(Static::Array(ArrayType::new_any())),
+            PartiqlShapeBuilder::init_or_get().new_static(Static::Bag(BagType::new_any())),
         ])];
 
         match *self {

partiql-eval/src/eval/expr/datetime.rs

@@ -1,6 +1,6 @@
 use crate::eval::expr::{BindError, BindEvalExpr, EvalExpr};
 
-use partiql_types::TYPE_DATETIME;
+use partiql_types::type_datetime;
 use partiql_value::Value::Missing;
 use partiql_value::{DateTime, Value};
 
@@ -43,7 +43,7 @@ impl BindEvalExpr for EvalExtractFn {
         }
 
         let create = |f: fn(&DateTime) -> Value| {
-            UnaryValueExpr::create_typed::<{ STRICT }, _>([TYPE_DATETIME], args, move |value| {
+            UnaryValueExpr::create_typed::<{ STRICT }, _>([type_datetime!()], args, move |value| {
                 match value {
                     Value::DateTime(dt) => f(dt.as_ref()),
                     _ => Missing,