Skip to content

Commit b75fa82

Browse files
viiryaviirya
committed
Add decimal type
1 parent cb71fed commit b75fa82

File tree

1 file changed

+130
-117
lines changed

1 file changed

+130
-117
lines changed

‎crates/iceberg/src/arrow.rs

Lines changed: 130 additions & 117 deletions
Original file line numberDiff line numberDiff line change
@@ -447,6 +447,10 @@ impl ArrowSchemaVisitor for ArrowSchemaConverter {
447447
DataType::Int64 => Ok(Type::Primitive(PrimitiveType::Long)),
448448
DataType::Float32 => Ok(Type::Primitive(PrimitiveType::Float)),
449449
DataType::Float64 => Ok(Type::Primitive(PrimitiveType::Double)),
450+
DataType::Decimal128(p, s) => Ok(Type::Primitive(PrimitiveType::Decimal {
451+
precision: *p as u32,
452+
scale: *s as u32,
453+
})),
450454
DataType::Date32 => Ok(Type::Primitive(PrimitiveType::Date)),
451455
DataType::Time64(unit) if unit == &TimeUnit::Microsecond => {
452456
Ok(Type::Primitive(PrimitiveType::Time))
@@ -529,128 +533,131 @@ mod tests {
529533

530534
let r#struct = DataType::Struct(fields);
531535

532-
let schema = ArrowSchema::new(vec![
533-
Field::new("a", DataType::Int32, false).with_metadata(HashMap::from([(
534-
ARROW_FIELD_ID_KEY.to_string(),
535-
"2".to_string(),
536-
)])),
537-
Field::new("b", DataType::Int64, false).with_metadata(HashMap::from([(
538-
ARROW_FIELD_ID_KEY.to_string(),
539-
"1".to_string(),
540-
)])),
541-
Field::new("c", DataType::Utf8, false).with_metadata(HashMap::from([(
542-
ARROW_FIELD_ID_KEY.to_string(),
543-
"3".to_string(),
544-
)])),
545-
Field::new("n", DataType::LargeUtf8, false).with_metadata(HashMap::from([(
546-
ARROW_FIELD_ID_KEY.to_string(),
547-
"21".to_string(),
548-
)])),
549-
Field::new("d", DataType::Timestamp(TimeUnit::Microsecond, None), true).with_metadata(
550-
HashMap::from([(ARROW_FIELD_ID_KEY.to_string(), "4".to_string())]),
551-
),
552-
Field::new("e", DataType::Boolean, true).with_metadata(HashMap::from([(
553-
ARROW_FIELD_ID_KEY.to_string(),
554-
"6".to_string(),
555-
)])),
556-
Field::new("f", DataType::Float32, false).with_metadata(HashMap::from([(
557-
ARROW_FIELD_ID_KEY.to_string(),
558-
"5".to_string(),
559-
)])),
560-
Field::new("g", DataType::Float64, false).with_metadata(HashMap::from([(
561-
ARROW_FIELD_ID_KEY.to_string(),
562-
"7".to_string(),
563-
)])),
564-
Field::new("h", DataType::Date32, false).with_metadata(HashMap::from([(
565-
ARROW_FIELD_ID_KEY.to_string(),
566-
"8".to_string(),
567-
)])),
568-
Field::new("i", DataType::Time64(TimeUnit::Microsecond), false).with_metadata(
569-
HashMap::from([(ARROW_FIELD_ID_KEY.to_string(), "9".to_string())]),
570-
),
571-
Field::new(
572-
"j",
573-
DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())),
574-
false,
575-
)
576-
.with_metadata(HashMap::from([(
577-
ARROW_FIELD_ID_KEY.to_string(),
578-
"10".to_string(),
579-
)])),
580-
Field::new(
581-
"k",
582-
DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),
583-
false,
584-
)
585-
.with_metadata(HashMap::from([(
586-
ARROW_FIELD_ID_KEY.to_string(),
587-
"12".to_string(),
588-
)])),
589-
Field::new("l", DataType::Binary, false).with_metadata(HashMap::from([(
590-
ARROW_FIELD_ID_KEY.to_string(),
591-
"13".to_string(),
592-
)])),
593-
Field::new("o", DataType::LargeBinary, false).with_metadata(HashMap::from([(
594-
ARROW_FIELD_ID_KEY.to_string(),
595-
"22".to_string(),
596-
)])),
597-
Field::new("m", DataType::FixedSizeBinary(10), false).with_metadata(HashMap::from([(
598-
ARROW_FIELD_ID_KEY.to_string(),
599-
"11".to_string(),
600-
)])),
601-
Field::new(
602-
"list",
603-
DataType::List(Arc::new(
604-
Field::new("element", DataType::Int32, false).with_metadata(HashMap::from([(
605-
ARROW_FIELD_ID_KEY.to_string(),
606-
"15".to_string(),
607-
)])),
608-
)),
609-
true,
610-
)
611-
.with_metadata(HashMap::from([(
612-
ARROW_FIELD_ID_KEY.to_string(),
613-
"14".to_string(),
614-
)])),
615-
Field::new(
616-
"large_list",
617-
DataType::LargeList(Arc::new(
618-
Field::new("element", DataType::Utf8, false).with_metadata(HashMap::from([(
536+
let schema =
537+
ArrowSchema::new(vec![
538+
Field::new("a", DataType::Int32, false).with_metadata(HashMap::from([(
539+
ARROW_FIELD_ID_KEY.to_string(),
540+
"2".to_string(),
541+
)])),
542+
Field::new("b", DataType::Int64, false).with_metadata(HashMap::from([(
543+
ARROW_FIELD_ID_KEY.to_string(),
544+
"1".to_string(),
545+
)])),
546+
Field::new("c", DataType::Utf8, false).with_metadata(HashMap::from([(
547+
ARROW_FIELD_ID_KEY.to_string(),
548+
"3".to_string(),
549+
)])),
550+
Field::new("n", DataType::LargeUtf8, false).with_metadata(HashMap::from([(
551+
ARROW_FIELD_ID_KEY.to_string(),
552+
"21".to_string(),
553+
)])),
554+
Field::new("d", DataType::Timestamp(TimeUnit::Microsecond, None), true)
555+
.with_metadata(HashMap::from([(
619556
ARROW_FIELD_ID_KEY.to_string(),
620-
"23".to_string(),
557+
"4".to_string(),
621558
)])),
559+
Field::new("e", DataType::Boolean, true).with_metadata(HashMap::from([(
560+
ARROW_FIELD_ID_KEY.to_string(),
561+
"6".to_string(),
562+
)])),
563+
Field::new("f", DataType::Float32, false).with_metadata(HashMap::from([(
564+
ARROW_FIELD_ID_KEY.to_string(),
565+
"5".to_string(),
566+
)])),
567+
Field::new("g", DataType::Float64, false).with_metadata(HashMap::from([(
568+
ARROW_FIELD_ID_KEY.to_string(),
569+
"7".to_string(),
570+
)])),
571+
Field::new("p", DataType::Decimal128(10, 2), false).with_metadata(HashMap::from([
572+
(ARROW_FIELD_ID_KEY.to_string(), "27".to_string()),
573+
])),
574+
Field::new("h", DataType::Date32, false).with_metadata(HashMap::from([(
575+
ARROW_FIELD_ID_KEY.to_string(),
576+
"8".to_string(),
577+
)])),
578+
Field::new("i", DataType::Time64(TimeUnit::Microsecond), false).with_metadata(
579+
HashMap::from([(ARROW_FIELD_ID_KEY.to_string(), "9".to_string())]),
580+
),
581+
Field::new(
582+
"j",
583+
DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())),
584+
false,
585+
)
586+
.with_metadata(HashMap::from([(
587+
ARROW_FIELD_ID_KEY.to_string(),
588+
"10".to_string(),
589+
)])),
590+
Field::new(
591+
"k",
592+
DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),
593+
false,
594+
)
595+
.with_metadata(HashMap::from([(
596+
ARROW_FIELD_ID_KEY.to_string(),
597+
"12".to_string(),
598+
)])),
599+
Field::new("l", DataType::Binary, false).with_metadata(HashMap::from([(
600+
ARROW_FIELD_ID_KEY.to_string(),
601+
"13".to_string(),
602+
)])),
603+
Field::new("o", DataType::LargeBinary, false).with_metadata(HashMap::from([(
604+
ARROW_FIELD_ID_KEY.to_string(),
605+
"22".to_string(),
606+
)])),
607+
Field::new("m", DataType::FixedSizeBinary(10), false).with_metadata(HashMap::from(
608+
[(ARROW_FIELD_ID_KEY.to_string(), "11".to_string())],
622609
)),
623-
true,
624-
)
625-
.with_metadata(HashMap::from([(
626-
ARROW_FIELD_ID_KEY.to_string(),
627-
"24".to_string(),
628-
)])),
629-
Field::new(
630-
"fixed_list",
631-
DataType::FixedSizeList(
632-
Arc::new(
633-
Field::new("element", DataType::Binary, false).with_metadata(
634-
HashMap::from([(ARROW_FIELD_ID_KEY.to_string(), "26".to_string())]),
610+
Field::new(
611+
"list",
612+
DataType::List(Arc::new(
613+
Field::new("element", DataType::Int32, false).with_metadata(HashMap::from(
614+
[(ARROW_FIELD_ID_KEY.to_string(), "15".to_string())],
615+
)),
616+
)),
617+
true,
618+
)
619+
.with_metadata(HashMap::from([(
620+
ARROW_FIELD_ID_KEY.to_string(),
621+
"14".to_string(),
622+
)])),
623+
Field::new(
624+
"large_list",
625+
DataType::LargeList(Arc::new(
626+
Field::new("element", DataType::Utf8, false).with_metadata(HashMap::from(
627+
[(ARROW_FIELD_ID_KEY.to_string(), "23".to_string())],
628+
)),
629+
)),
630+
true,
631+
)
632+
.with_metadata(HashMap::from([(
633+
ARROW_FIELD_ID_KEY.to_string(),
634+
"24".to_string(),
635+
)])),
636+
Field::new(
637+
"fixed_list",
638+
DataType::FixedSizeList(
639+
Arc::new(
640+
Field::new("element", DataType::Binary, false).with_metadata(
641+
HashMap::from([(ARROW_FIELD_ID_KEY.to_string(), "26".to_string())]),
642+
),
635643
),
644+
10,
636645
),
637-
10,
638-
),
639-
true,
640-
)
641-
.with_metadata(HashMap::from([(
642-
ARROW_FIELD_ID_KEY.to_string(),
643-
"25".to_string(),
644-
)])),
645-
Field::new("map", map, false).with_metadata(HashMap::from([(
646-
ARROW_FIELD_ID_KEY.to_string(),
647-
"16".to_string(),
648-
)])),
649-
Field::new("struct", r#struct, false).with_metadata(HashMap::from([(
650-
ARROW_FIELD_ID_KEY.to_string(),
651-
"17".to_string(),
652-
)])),
653-
]);
646+
true,
647+
)
648+
.with_metadata(HashMap::from([(
649+
ARROW_FIELD_ID_KEY.to_string(),
650+
"25".to_string(),
651+
)])),
652+
Field::new("map", map, false).with_metadata(HashMap::from([(
653+
ARROW_FIELD_ID_KEY.to_string(),
654+
"16".to_string(),
655+
)])),
656+
Field::new("struct", r#struct, false).with_metadata(HashMap::from([(
657+
ARROW_FIELD_ID_KEY.to_string(),
658+
"17".to_string(),
659+
)])),
660+
]);
654661
let schema = Arc::new(schema);
655662
let result = arrow_schema_to_schema(&schema).unwrap();
656663

@@ -706,6 +713,12 @@ mod tests {
706713
"required":true,
707714
"type":"double"
708715
},
716+
{
717+
"id":27,
718+
"name":"p",
719+
"required":true,
720+
"type":"decimal(10,2)"
721+
},
709722
{
710723
"id":8,
711724
"name":"h",

0 commit comments

Comments
 (0)