From 6b05270028285abdb96b8d08e3a1e518882e8510 Mon Sep 17 00:00:00 2001
From: Alex Huszagh <ahuszagh@gmail.com>
Date: Wed, 19 May 2021 16:36:02 -0500
Subject: [PATCH] Add support for tokenized floats.
---
extras/simple-bench/Cargo.toml | 4 +
extras/simple-bench/src/main.rs | 94 ++++++++-
src/decimal.rs | 101 +++++++---
src/lib.rs | 30 +++
src/number.rs | 96 +++++++--
src/parse.rs | 61 ++++--
src/simple.rs | 6 +-
tests/test_json.rs | 336 ++++++++++++++++++++++++++++++++
8 files changed, 669 insertions(+), 59 deletions(-)
create mode 100644 tests/test_json.rs
diff --git a/extras/simple-bench/Cargo.toml b/extras/simple-bench/Cargo.toml
index 5cd59c8..a42a314 100644
--- a/extras/simple-bench/Cargo.toml
+++ b/extras/simple-bench/Cargo.toml
@@ -14,3 +14,7 @@ anyhow = "1.0"
lexical = "5.2"
lexical-core = "0.7"
fastrand = "1.4"
+
+[features]
+default = []
+use_tokenized = []
diff --git a/extras/simple-bench/src/main.rs b/extras/simple-bench/src/main.rs
index 9428fad..438f197 100644
--- a/extras/simple-bench/src/main.rs
+++ b/extras/simple-bench/src/main.rs
@@ -108,6 +108,8 @@ fn run_bench<T: FastFloat, F: Fn(&str) -> T>(
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
enum Method {
FastFloat,
+ #[cfg(feature = "use_tokenized")]
+ FastFloatTokenized,
Lexical,
FromStr,
}
@@ -120,10 +122,87 @@ fn type_str(float32: bool) -> &'static str {
}
}
+#[inline]
+#[cfg(feature = "use_tokenized")]
+fn parse_sign<'a>(s: &'a str) -> (bool, &'a str) {
+ match s.as_bytes().get(0) {
+ Some(&b'+') => (false, &s[1..]),
+ Some(&b'-') => (true, &s[1..]),
+ _ => (false, s),
+ }
+}
+
+#[inline]
+#[cfg(feature = "use_tokenized")]
+fn decimal_index(s: &str) -> Option<usize> {
+ s.as_bytes().iter().position(|&c| c == b'.')
+}
+
+#[inline]
+#[cfg(feature = "use_tokenized")]
+fn exponent_index(s: &str) -> Option<usize> {
+ s.as_bytes().iter().position(|&c| c == b'e' || c == b'E')
+}
+
+#[inline]
+#[cfg(feature = "use_tokenized")]
+fn split_index<'a>(s: &'a str, index: usize) -> (&'a str, &'a str) {
+ let (lead, trail) = s.as_bytes().split_at(index);
+ let trail = &trail[1..];
+ use std::str;
+ unsafe {
+ (str::from_utf8_unchecked(lead), str::from_utf8_unchecked(trail))
+ }
+}
+
+#[inline]
+#[cfg(feature = "use_tokenized")]
+fn split_end<'a>(s: &'a str) -> (&'a str, &'a str) {
+ let (lead, trail) = s.as_bytes().split_at(s.len());
+ use std::str;
+ unsafe {
+ (str::from_utf8_unchecked(lead), str::from_utf8_unchecked(trail))
+ }
+}
+
+#[inline]
+#[cfg(feature = "use_tokenized")]
+fn parse_exponent(s: &str) -> i64 {
+ s.parse::<i64>().unwrap()
+}
+
+#[inline]
+#[cfg(feature = "use_tokenized")]
+fn tokenize<'a>(s: &'a str) -> (&'a str, &'a str, i64, bool) {
+ let (negative, s) = parse_sign(s);
+ if let Some(index) = decimal_index(s) {
+ let (i, rest) = split_index(s, index);
+ if let Some(index) = exponent_index(s) {
+ let (f, exp) = split_index(rest, index);
+ let exp = parse_exponent(exp);
+ (i, f, exp, negative)
+ } else {
+ (i, rest, 0, negative)
+ }
+ } else {
+ if let Some(index) = exponent_index(s) {
+ let (i, exp) = split_index(s, index);
+ let (i, f) = split_end(i);
+ let exp = parse_exponent(exp);
+ (i, f, exp, negative)
+ } else {
+ let (i, f) = split_end(s);
+ (i, f, 0, negative)
+ }
+ }
+}
+
impl Method {
pub fn name(&self) -> &'static str {
match self {
Self::FastFloat => "fast-float",
+ #[cfg(feature = "use_tokenized")]
+ Self::FastFloatTokenized => "fast-float-tokenized",
Self::Lexical => "lexical",
Self::FromStr => "from_str",
}
@@ -140,6 +219,11 @@ impl Method {
Self::FastFloat => run_bench(data, repeat, |s: &str| {
fast_float::parse_partial::<T, _>(s).unwrap_or_default().0
}),
+ #[cfg(feature = "use_tokenized")]
+ Self::FastFloatTokenized => run_bench(data, repeat, |s: &str| {
+ let (i, f, e, n) = tokenize(s);
+ fast_float::parse_from_parts::<T, _>(i, f, e, n)
+ }),
Self::Lexical => run_bench(data, repeat, |s: &str| {
lexical_core::parse_partial::<T>(s.as_bytes())
.unwrap_or_default()
@@ -165,7 +249,15 @@ impl Method {
}
pub fn all() -> &'static [Self] {
- &[Method::FastFloat, Method::Lexical, Method::FromStr]
+ #[cfg(feature = "use_tokenized")]
+ {
+ &[Method::FastFloat, Method::FastFloatTokenized, Method::Lexical, Method::FromStr]
+ }
+
+ #[cfg(not(feature = "use_tokenized"))]
+ {
+ &[Method::FastFloat, Method::Lexical, Method::FromStr]
+ }
}
}
diff --git a/src/decimal.rs b/src/decimal.rs
index c36d6d3..9d612e1 100644
--- a/src/decimal.rs
+++ b/src/decimal.rs
@@ -187,41 +187,37 @@ impl Decimal {
}
#[inline]
-pub fn parse_decimal(mut s: &[u8]) -> Decimal {
- // can't fail since it follows a call to parse_number
- let mut d = Decimal::default();
- let start = s;
- let c = s.get_first();
- d.negative = c == b'-';
- if c == b'-' || c == b'+' {
- s = s.advance(1);
+fn parse_fractional<'a>(mut s: &'a [u8], d: &mut Decimal) -> &'a [u8] {
+ let first = s;
+ if d.num_digits == 0 {
+ s = s.skip_chars(b'0');
}
- s = s.skip_chars(b'0');
- parse_digits(&mut s, |digit| d.try_add_digit(digit));
- if s.check_first(b'.') {
- s = s.advance(1);
- let first = s;
- if d.num_digits == 0 {
- s = s.skip_chars(b'0');
- }
- if cfg!(target_endian = "little") {
- while s.len() >= 8 && d.num_digits + 8 < Decimal::MAX_DIGITS {
- let v = s.read_u64();
- if !is_8digits_le(v) {
- break;
- }
- d.digits[d.num_digits..].write_u64(v - 0x3030_3030_3030_3030);
- d.num_digits += 8;
- s = s.advance(8);
+ if cfg!(target_endian = "little") {
+ while s.len() >= 8 && d.num_digits + 8 < Decimal::MAX_DIGITS {
+ let v = s.read_u64();
+ if !is_8digits_le(v) {
+ break;
}
+ d.digits[d.num_digits..].write_u64(v - 0x3030_3030_3030_3030);
+ d.num_digits += 8;
+ s = s.advance(8);
}
- parse_digits(&mut s, |digit| d.try_add_digit(digit));
- d.decimal_point = s.len() as i32 - first.len() as i32;
}
+ parse_digits(&mut s, |digit| d.try_add_digit(digit));
+ d.decimal_point = s.len() as i32 - first.len() as i32;
+
+ s
+}
+
+#[inline]
+fn trim_zeros<'a, Iter>(iter: Iter, d: &mut Decimal)
+where
+ Iter: Iterator<Item=&'a u8>
+{
if d.num_digits != 0 {
// Ignore the trailing zeros if there are any
let mut n_trailing_zeros = 0;
- for &c in start[..(start.len() - s.len())].iter().rev() {
+ for &c in iter {
if c == b'0' {
n_trailing_zeros += 1;
} else if c != b'.' {
@@ -236,6 +232,51 @@ pub fn parse_decimal(mut s: &[u8]) -> Decimal {
d.num_digits = Decimal::MAX_DIGITS;
}
}
+}
+
+#[inline]
+fn add_zero_digits(d: &mut Decimal) {
+ for i in d.num_digits..Decimal::MAX_DIGITS_WITHOUT_OVERFLOW {
+ d.digits[i] = 0;
+ }
+}
+
+#[inline]
+pub fn parse_decimal_from_parts(mut i: &[u8], f: &[u8], e: i64, negative: bool) -> Decimal {
+ // can't fail since it follows a call to parse_number
+ let mut d = Decimal::default();
+
+ // Integral
+ let i_start = i;
+ d.negative = negative;
+ i = i.skip_chars(b'0');
+ parse_digits(&mut i, |digit| d.try_add_digit(digit));
+
+ parse_fractional(f, &mut d);
+ trim_zeros(i_start.iter().chain(f.iter()).rev(), &mut d);
+ d.decimal_point += e as i32;
+ add_zero_digits(&mut d);
+
+ d
+}
+
+#[inline]
+pub fn parse_decimal(mut s: &[u8]) -> Decimal {
+ // can't fail since it follows a call to parse_number
+ let mut d = Decimal::default();
+ let start = s;
+ let c = s.get_first();
+ d.negative = c == b'-';
+ if c == b'-' || c == b'+' {
+ s = s.advance(1);
+ }
+ s = s.skip_chars(b'0');
+ parse_digits(&mut s, |digit| d.try_add_digit(digit));
+ if s.check_first(b'.') {
+ s = s.advance(1);
+ s = parse_fractional(s, &mut d);
+ }
+ trim_zeros(start[..(start.len() - s.len())].iter().rev(), &mut d);
if s.check_first2(b'e', b'E') {
s = s.advance(1);
let mut neg_exp = false;
@@ -253,9 +294,7 @@ pub fn parse_decimal(mut s: &[u8]) -> Decimal {
});
d.decimal_point += if neg_exp { -exp_num } else { exp_num };
}
- for i in d.num_digits..Decimal::MAX_DIGITS_WITHOUT_OVERFLOW {
- d.digits[i] = 0;
- }
+ add_zero_digits(&mut d);
d
}
diff --git a/src/lib.rs b/src/lib.rs
index aef86be..5f30dd4 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -105,6 +105,21 @@ pub trait FastFloat: float::Float {
fn parse_float_partial<S: AsRef<[u8]>>(s: S) -> Result<(Self, usize)> {
parse::parse_float(s.as_ref()).ok_or(Error)
}
+
+ /// Parse a pre-tokenized decimal number from string into float.
+ ///
+ /// This assumes the float has already been tokenized into valid
+ /// integral and fractional components, and has parsed an optional
+ /// exponent notation.
+ ///
+ /// It is up to you to validate and tokenize the input: although
+ /// this will not error, this might truncate the significant
+ /// digits as soon as an invalid digit is found. This does not
+ /// handle special values, such as NaN, INF, or Infinity.
+ #[inline]
+ fn parse_from_parts<S: AsRef<[u8]>>(integral: S, fractional: S, exponent: i64, negative: bool) -> Self {
+ parse::parse_from_parts(integral.as_ref(), fractional.as_ref(), exponent, negative)
+ }
}
impl FastFloat for f32 {}
@@ -134,3 +149,18 @@ pub fn parse<T: FastFloat, S: AsRef<[u8]>>(s: S) -> Result<T> {
pub fn parse_partial<T: FastFloat, S: AsRef<[u8]>>(s: S) -> Result<(T, usize)> {
T::parse_float_partial(s)
}
+
+/// Parse a pre-tokenized decimal number from string into float.
+///
+/// This assumes the float has already been tokenized into valid
+/// integral and fractional components, and has parsed an optional
+/// exponent notation.
+///
+/// It is up to you to validate and tokenize the input: although
+/// this will not error, this might truncate the significant
+/// digits as soon as an invalid digit is found. This does not
+/// handle special values, such as NaN, INF, or Infinity.
+#[inline]
+pub fn parse_from_parts<T: FastFloat, S: AsRef<[u8]>>(integral: S, fractional: S, exponent: i64, negative: bool) -> T {
+ T::parse_from_parts(integral.as_ref(), fractional.as_ref(), exponent, negative)
+}
diff --git a/src/number.rs b/src/number.rs
index 194cbf6..d40d657 100644
--- a/src/number.rs
+++ b/src/number.rs
@@ -148,6 +148,88 @@ fn parse_scientific(s: &mut AsciiStr<'_>) -> i64 {
}
}
+#[inline]
+fn parse_integral(s: &mut AsciiStr, start: &AsciiStr, mantissa: &mut u64) -> isize {
+ try_parse_digits(s, mantissa);
+ s.offset_from(start)
+}
+
+#[inline]
+fn parse_fractional(s: &mut AsciiStr, start: &AsciiStr, mantissa: &mut u64) -> isize {
+ try_parse_8digits_le(s, mantissa);
+ try_parse_digits(s, mantissa);
+ s.offset_from(start)
+}
+
+#[inline]
+fn trim_leading_zeros(mut n_digits: isize, mut p: AsciiStr) -> isize {
+ n_digits -= 19;
+ while p.check_first_either(b'0', b'.') {
+ n_digits -= p.first().saturating_sub(b'0' - 1) as isize; // '0' = b'.' + 2
+ p.step();
+ }
+ n_digits
+}
+
+#[inline]
+pub fn parse_number_from_parts(i: &[u8], f: &[u8], e: i64, negative: bool) -> Option<Number> {
+
+ let mut mantissa = 0_u64;
+ let mut i = AsciiStr::new(i);
+ let i_start = i;
+ let mut n_digits = parse_integral(&mut i, &i_start, &mut mantissa);
+
+ let mut f = AsciiStr::new(f);
+ let f_start = f;
+ let int_end = i;
+ let n_after_dot = parse_fractional(&mut f, &f_start, &mut mantissa);
+ n_digits += n_after_dot;
+ let mut exponent = e - n_after_dot as i64;
+
+ if n_digits == 0 {
+ return None;
+ }
+
+ // handle uncommon case with many digits
+ if n_digits <= 19 {
+ return Some(
+ Number {
+ exponent,
+ mantissa,
+ negative,
+ many_digits: false,
+ }
+ );
+ }
+
+ let mut many_digits = false;
+ n_digits = trim_leading_zeros(n_digits, i_start);
+ if n_digits > 0 {
+ // at this point we have more than 19 significant digits, let's try again
+ many_digits = true;
+ mantissa = 0;
+ let mut i = i_start;
+ try_parse_19digits(&mut i, &mut mantissa);
+ exponent = if mantissa >= MIN_19DIGIT_INT {
+ int_end.offset_from(&i) // big int
+ } else {
+ let mut f = f_start;
+ try_parse_19digits(&mut f, &mut mantissa);
+ -f.offset_from(&f_start)
+ } as i64;
+ exponent += e; // add back the explicit part
+ }
+
+ Some(
+ Number {
+ exponent,
+ mantissa,
+ negative,
+ many_digits,
+ },
+ )
+}
+
#[inline]
pub fn parse_number(s: &[u8]) -> Option<(Number, usize)> {
debug_assert!(!s.is_empty());
@@ -170,8 +252,7 @@ pub fn parse_number(s: &[u8]) -> Option<(Number, usize)> {
// parse initial digits before dot
let mut mantissa = 0_u64;
let digits_start = s;
- try_parse_digits(&mut s, &mut mantissa);
- let mut n_digits = s.offset_from(&digits_start);
+ let mut n_digits = parse_integral(&mut s, &digits_start, &mut mantissa);
// handle dot with the following digits
let mut n_after_dot = 0;
@@ -180,9 +261,7 @@ pub fn parse_number(s: &[u8]) -> Option<(Number, usize)> {
if s.check_first(b'.') {
s.step();
let before = s;
- try_parse_8digits_le(&mut s, &mut mantissa);
- try_parse_digits(&mut s, &mut mantissa);
- n_after_dot = s.offset_from(&before);
+ n_after_dot = parse_fractional(&mut s, &before, &mut mantissa);
exponent = -n_after_dot as i64;
}
@@ -213,13 +292,8 @@ pub fn parse_number(s: &[u8]) -> Option<(Number, usize)> {
));
}
- n_digits -= 19;
let mut many_digits = false;
- let mut p = digits_start;
- while p.check_first_either(b'0', b'.') {
- n_digits -= p.first().saturating_sub(b'0' - 1) as isize; // '0' = b'.' + 2
- p.step();
- }
+ n_digits = trim_leading_zeros(n_digits, digits_start);
if n_digits > 0 {
// at this point we have more than 19 significant digits, let's try again
many_digits = true;
diff --git a/src/parse.rs b/src/parse.rs
index 9c592d4..465d8fa 100644
--- a/src/parse.rs
+++ b/src/parse.rs
@@ -1,8 +1,52 @@
use crate::binary::compute_float;
+use crate::common::AdjustedMantissa;
+use crate::decimal::{parse_decimal, parse_decimal_from_parts};
use crate::float::Float;
-use crate::number::{parse_inf_nan, parse_number};
+use crate::number::{Number, parse_inf_nan, parse_number, parse_number_from_parts};
use crate::simple::parse_long_mantissa;
+#[inline]
+fn lemire<F: Float>(num: &Number) -> AdjustedMantissa {
+ let mut am = compute_float::<F>(num.exponent, num.mantissa);
+ if num.many_digits && am != compute_float::<F>(num.exponent, num.mantissa + 1) {
+ am.power2 = -1;
+ }
+ am
+}
+
+#[inline]
+fn to_float<F: Float>(am: AdjustedMantissa, negative: bool) -> F {
+ let mut word = am.mantissa;
+ word |= (am.power2 as u64) << F::MANTISSA_EXPLICIT_BITS;
+ if negative {
+ word |= 1_u64 << F::SIGN_INDEX;
+ }
+ F::from_u64_bits(word)
+}
+
+#[inline]
+pub fn parse_from_parts<F: Float>(i: &[u8], f: &[u8], e: i64, negative: bool) -> F {
+ if i.is_empty() && f.is_empty() {
+ return F::from_u64(0);
+ }
+
+ let num = match parse_number_from_parts(i, f, e, negative) {
+ Some(n) => n,
+ None => return F::from_u64(0),
+ };
+ if let Some(value) = num.try_fast_path::<F>() {
+ return value;
+ }
+
+ let mut am = lemire::<F>(&num);
+ if am.power2 < 0 {
+ am = parse_long_mantissa::<F>(parse_decimal_from_parts(i, f, e, negative));
+ }
+
+ to_float(am, num.negative)
+}
+
+
#[inline]
pub fn parse_float<F: Float>(s: &[u8]) -> Option<(F, usize)> {
if s.is_empty() {
@@ -17,18 +61,11 @@ pub fn parse_float<F: Float>(s: &[u8]) -> Option<(F, usize)> {
return Some((value, rest));
}
- let mut am = compute_float::<F>(num.exponent, num.mantissa);
- if num.many_digits && am != compute_float::<F>(num.exponent, num.mantissa + 1) {
- am.power2 = -1;
- }
+ let mut am = lemire::<F>(&num);
if am.power2 < 0 {
- am = parse_long_mantissa::<F>(s);
+ am = parse_long_mantissa::<F>(parse_decimal(s));
}
- let mut word = am.mantissa;
- word |= (am.power2 as u64) << F::MANTISSA_EXPLICIT_BITS;
- if num.negative {
- word |= 1_u64 << F::SIGN_INDEX;
- }
- Some((F::from_u64_bits(word), rest))
+ let flt = to_float(am, num.negative);
+ Some((flt, rest))
}
diff --git a/src/simple.rs b/src/simple.rs
index cec1183..226d244 100644
--- a/src/simple.rs
+++ b/src/simple.rs
@@ -1,9 +1,9 @@
use crate::common::AdjustedMantissa;
-use crate::decimal::{parse_decimal, Decimal};
+use crate::decimal::Decimal;
use crate::float::Float;
#[inline]
-pub fn parse_long_mantissa<F: Float>(s: &[u8]) -> AdjustedMantissa {
+pub fn parse_long_mantissa<F: Float>(mut d: Decimal) -> AdjustedMantissa {
const MAX_SHIFT: usize = 60;
const NUM_POWERS: usize = 19;
const POWERS: [u8; 19] = [
@@ -21,8 +21,6 @@ pub fn parse_long_mantissa<F: Float>(s: &[u8]) -> AdjustedMantissa {
let am_zero = AdjustedMantissa::zero_pow2(0);
let am_inf = AdjustedMantissa::zero_pow2(F::INFINITE_POWER);
- let mut d = parse_decimal(s);
-
if d.num_digits == 0 || d.decimal_point < -324 {
return am_zero;
} else if d.decimal_point >= 310 {
diff --git a/tests/test_json.rs b/tests/test_json.rs
new file mode 100644
index 0000000..cf8790c
--- /dev/null
+++ b/tests/test_json.rs
@@ -0,0 +1,336 @@
+use std::string::ToString;
+
+macro_rules! check_float {
+ ($i:expr, $f:expr, $e:expr, $n:expr, $ty:ty) => {{
+ let mut s = String::new();
+ if $n {
+ s.push('-');
+ }
+ s.push_str($i);
+ if $f.len() > 0 {
+ s.push('.');
+ s.push_str($f);
+ }
+ if $e != 0 {
+ s.push('e');
+ s.push_str(&$e.to_string());
+ }
+ let expected = fast_float::parse::<$ty, _>(s).unwrap();
+ let result = fast_float::parse_from_parts::<$ty, _>($i, $f, $e, $n);
+ assert_eq!(expected, result);
+ }};
+}
+
+macro_rules! check_f32 {
+ ($i:expr, $f:expr, $e:expr, $n:expr) => {{
+ check_float!($i, $f, $e, $n, f32);
+ }};
+}
+
+macro_rules! check_f64 {
+ ($i:expr, $f:expr, $e:expr, $n:expr) => {{
+ check_float!($i, $f, $e, $n, f64);
+ }};
+}
+
+fn append_zeros(s: impl AsRef<str>, n: usize) -> String {
+ let mut s = String::from(s.as_ref());
+ for _ in 0..n {
+ s.push('0');
+ }
+ s
+}
+
+#[test]
+fn test_tokenized_f64() {
+ // Inf
+ check_f64!("1234456789012345678901234567890", "0", 2147483647, false);
+ check_f64!("1", "832312213213213232132132143451234453123412321321312", 308, false);
+ check_f64!("2", "0", 2147483647, false);
+ check_f64!("2", "0", 3000, false);
+ check_f64!("1", "8", 308, false);
+ check_f64!("1", "9", 308, false);
+
+ // Negative inf
+ check_f64!("2139879401095466344511101915470454744", "9813888656856943", 272, true);
+
+ // Long
+ check_f64!(
+ "9355950000000000000",
+ "\
+ 000000000000000000000000000000000018446744073709551616000001\
+ 84467440737095516161844674407370955161407370955161618446744073709551616000184467\
+ 44073709551616600000184467440737095516161844674407370955161407370955161618446744\
+ 07370955161600018446744073709551616018446744073709556744516161844674407370955161\
+ 40737095516161844674407370955161600018446744073709551616018446744073709551611616\
+ 00018446744073709500184467440737095516160018446744073709551616001844674407370955\
+ 11681644674407370955161600018440737095516160184467440737095516161844674407370955\
+ 16160001844674407536910751601611616000184467440737095001844674407370955161600184\
+ 46744073709551616001844674407370955161618446744073709551616000184495516161844674\
+ 4073709551616000184467440753691075160018446744073709",
+ 0,
+ false
+ );
+ check_f64!(
+ "2",
+ "\
+ 225073858507202124188701479202220329072405282794390378143031338374351073192441\
+ 94686754406432563881851382188218502438069999947733013005649884107791928741341929\
+ 29720097048195199306799329096904278406473168204156592672863293363047467012331685\
+ 29834221527445172608358596545663192828352447877877998943107797838336991592885945\
+ 55213714181128458251145584319223079897504395086859412457230891738946169368372321\
+ 19137365897797772328669884035639025104444303545739673370658398105542045669382465\
+ 84137476071559811765738776267476659123871999319040063173347090030127901881752034\
+ 47190250028061277777916798391090578584006464715943810511489154282775041174682194\
+ 13395246668250343130618158782937900420539237507208336669324158000275839111885418\
+ 8641513168478436313080237596295773983001708984375",
+ -308,
+ false
+ );
+ check_f64!(
+ "\
+ 14384566631413902735261182076422355811832278452463312311626366537903681520913941\
+ 96930365828634687637948157940776599182791387527135353034738357134110310609455693\
+ 90082419354977279201654318268051974058035436546798544018359870131225762454556233\
+ 13970183299286131961255902741877200739148180625308303165331580986249841188892982\
+ 81371812288789537310599037529113415438738954894752124724983067241108764488346454\
+ 37669901867307840475112141480493722424080599312381693232622368309077056159757045\
+ 77939329858261626042558845291341263962822021265262533893834218067279545885255961\
+ 14379801269094096329805054803089299736996870951258573010877404407451953846698609\
+ 19821392688269207855703322826525930548119852605981316446918758669325733577952202\
+ 04076454986842633399219052275566166981299674128912822316855046606712779271982900\
+ 09824680186319750978665734576683784255802269708917361719466043175201158849097881\
+ 37047711185017157986905601606166617302905958843377601564443970505037755427769614\
+ 39282780934537928038462527159660167332226464423828921239400524413468224297215938\
+ 84378212558701004356924243030059517489346646577724622498919752597382095222500311\
+ 12418182351225107135618176937657765139002829779615620881537508915912839494571051\
+ 58613344862671017974971111259092725051947928708896171797587034426080161433432621\
+ 59998149700606597792535574457560429226974273443630323818747730771316763398572110\
+ 87495998192373246307688452867739265415001026982223940199342748237651323138921235\
+ 35835735663769155726509168665536123661873789595549835667127670933729060301889762\
+ 20169058025354973622211666504549316958271880975697143546564469806791358707318873\
+ 07570838334500409015197406832583817753126695417740666139222980134999469594150993\
+ 5655355652985723782153570084089560139142231",
+ "\
+ 738475042362596875449154552392299548\
+ 94713816208169416867534067784380761312978044932336375902701297246698737092181681\
+ 31626587547265451210905455072402670004565947865409496052607224619378706306348749\
+ 91729398208026467698131898691830012167897399682179601734569071423681",
+ -733,
+ false
+ );
+ check_f64!(
+ "0",
+ "\
+ 000000000000000000000000000000000000000000000000000000000000000000000000000000\
+ 00000000000000000000000000000000000000000000000000000000000000000000000000000000\
+ 00000000000000000000000000000000000000000000000000000000000000000000000000000000\
+ 00000000000000000000000000000000000000000000000000000000000000000000044501477170\
+ 14402272114819593418263951869639092703291296046852219449644444042153891033059047\
+ 81627017582829831782607924221374017287738918929105531441481564124348675997628212\
+ 65346585071045737627442980259622449029037796981144446145705102663115100318287949\
+ 52795966823603998647925096578034214163701381261333311989876551545144031526125381\
+ 32666529513060001849177663286607555958373922409899478075565940981010216121988146\
+ 05258742579179000071675999344145086087205681577915435923018910334964869420614052\
+ 18289243144579760516365090360651414037721744226256159024466852576737244643007551\
+ 33324500796506867194913776884780053099639677097589658441378944337966219939673169\
+ 36280457084866613206797017728916080020698679408551343728867675409720757232455434\
+ 770912461317493580281734466552734375",
+ 0,
+ false
+ );
+ check_f64!(
+ "0",
+ "\
+ 000000000000000000000000000000000000000000000000000000000000000000000000000000\
+ 00000000000000000000000000000000000000000000000000000000000000000000000000000000\
+ 00000000000000000000000000000000000000000000000000000000000000000000000000000000\
+ 00000000000000000000000000000000000000000000000000000000000000000000022250738585\
+ 07200889024586876085859887650423112240959465493524802562440009228235695178775888\
+ 80375915526423097809504343120858773871583572918219930202943792242235598198275012\
+ 42041788969571311791082261043971979604000454897391938079198936081525613113376149\
+ 84204327175103362739154978273159414382813627511383860409424946494228631669542910\
+ 50802018159266421349966065178030950759130587198464239060686371020051087232827846\
+ 78843631944515866135041223479014792369585208321597621066375401613736583044193603\
+ 71477835530668283453563400507407304013560296804637591858316312422452159926254649\
+ 43008368518617194224176464551371354201322170313704965832101546540680353974179060\
+ 22589503023501937519773030945763173210852507299305089761582519159720757232455434\
+ 770912461317493580281734466552734375",
+ 0,
+ false
+ );
+
+ // General
+ check_f64!("9007199254740993", "0", 0, false);
+ check_f64!("9007199254740993", &append_zeros("0", 1000), 0, false);
+ check_f64!("10000000000000000000", "", 0, false);
+ check_f64!("10000000000000000000000000000001000000000000", "", 0, false);
+ check_f64!("10000000000000000000000000000000000000000001", "", 0, false);
+ check_f64!("1", "1920928955078125", -7, false);
+ check_f64!("1", "0000000000000006661338147750939242541790008544921875", 0, false);
+ check_f64!("1090544144181609348835077142190", "", 0, false);
+ check_f64!("2", "2250738585072013", -308, false);
+ check_f64!("92666518056446206563", "", 3, true);
+ check_f64!("42823146028335318693", "", -128, true);
+ check_f64!("90054602635948575728", "", 72, false);
+ check_f64!(
+ "1",
+ "\
+ 000000000000001885589208702234638701745660206917535153946435506630705583683732\
+ 21972569761144603605635692374830246134201063722058",
+ -309,
+ false
+ );
+ check_f64!("0", "", 2147483647, false);
+ check_f64!("2402844368454405395", "2", 2147483647, true);
+ check_f64!("2402844368454405395", "2", 2147483647, false);
+ check_f64!("7", "0420557077594588669468784357561207962098443483187940792729600000", 59, false);
+ check_f64!("7", "0420557077594588669468784357561207962098443483187940792729600000", 42, false);
+ check_f64!("1", "7339253062092163730578609458683877051596800000000000000000000000", 42, true);
+ check_f64!("2", "0972622234386619214559824785284023792871122537545728000000000000", 52, true);
+ check_f64!("1", "0001803374372191849407179462120053338028379051879898808320000000", 57, true);
+ check_f64!("1", "8607245283054342363818436991534856973992070520151142825984000000", 58, true);
+ check_f64!("1", "9189205311132686907264385602245237137907390376574976000000000000", 52, true);
+ check_f64!("2", "8184483231688951563253238886553506793085187889855201280000000000", 54, true);
+ check_f64!("1", "7664960224650106892054063261344555646357024359107788800000000000", 53, true);
+ check_f64!("2", "1470977154320536489471030463761883783915110400000000000000000000", 45, true);
+ check_f64!("4", "4900312744003159009338275160799498340862630046359789166919680000", 61, true);
+ check_f64!("1", "", 0, false);
+ check_f64!("1", "797693134862315700000000000000001", 308, false);
+ check_f64!("3", "", -324, false);
+ check_f64!("1", "00000006", 9, false);
+ check_f64!("4", "9406564584124653", -324, false);
+ check_f64!("4", "9406564584124654", -324, false);
+ check_f64!("2", "2250738585072009", -308, false);
+ check_f64!("2", "2250738585072014", -308, false);
+ check_f64!("1", "7976931348623157", 308, false);
+ check_f64!("1", "7976931348623158", 308, false);
+ check_f64!("4503599627370496", "5", 0, false);
+ check_f64!("4503599627475352", "5", 0, false);
+ check_f64!("4503599627475353", "5", 0, false);
+ check_f64!("2251799813685248", "25", 0, false);
+ check_f64!("1125899906842624", "125", 0, false);
+ check_f64!("1125899906842901", "875", 0, false);
+ check_f64!("2251799813685803", "75", 0, false);
+ check_f64!("4503599627370497", "5", 0, false);
+ check_f64!("45035996", "273704995", 0, false);
+ check_f64!("45035996", "273704985", 0, false);
+ check_f64!("1", "2345", 30, false);
+}
+
+#[test]
+fn test_tokenized_f32() {
+ // Inf
+ check_f32!("1234456789012345678901234567890", "", 2147483647, false);
+ check_f32!("2", "", 3000, false);
+ check_f32!("3", "5028234666", 38, false);
+
+ // General
+ let f1 = "\
+ 175494140627517859246175898662808184331245864732796240031385942718174675986064\
+ 7699724722770042717456817626953125";
+ check_f32!("1", f1, 0, false);
+ check_f32!("1", &append_zeros(f1, 655), 0, false);
+ check_f32!("1", &append_zeros(f1, 656), 0, false);
+ check_f32!("1", &append_zeros(f1, 1000), 0, false);
+ check_f32!("1", "00000006", 9, false);
+ check_f32!("1", "4012984643", -45, false);
+ check_f32!("1", "1754942107", -38, false);
+ check_f32!("1", "1754943508", -45, false);
+ check_f32!("0", "", 0, true);
+ check_f32!("1090544144181609348835077142190", "", 0, false);
+ check_f32!("1", "1754943508", -38, false);
+ check_f32!("30219", "0830078125", 0, false);
+ check_f32!("16252921", "5", 0, false);
+ check_f32!("5322519", "25", 0, false);
+ check_f32!("3900245", "875", 0, false);
+ check_f32!("1510988", "3125", 0, false);
+ check_f32!("782262", "28125", 0, false);
+ check_f32!("328381", "484375", 0, false);
+ check_f32!("156782", "0703125", 0, false);
+ check_f32!("85003", "24609375", 0, false);
+ check_f32!("43827", "048828125", 0, false);
+ check_f32!("17419", "6494140625", 0, false);
+ check_f32!("15498", "36376953125", 0, false);
+ check_f32!("6318", "580322265625", 0, false);
+ check_f32!("2525", "2840576171875", 0, false);
+ check_f32!("1370", "9265747070312", 0, false);
+ check_f32!("936", "3702087402344", 0, false);
+ check_f32!("411", "88682556152344", 0, false);
+ check_f32!("206", "50310516357422", 0, false);
+ check_f32!("124", "16878890991211", 0, false);
+ check_f32!("50", "811574935913086", 0, false);
+ check_f32!("17", "486443519592285", 0, false);
+ check_f32!("13", "91745138168335", 0, false);
+ check_f32!("7", "5464513301849365", 0, false);
+ check_f32!("2", "687217116355896", 0, false);
+ check_f32!("1", "1877630352973938", 0, false);
+ check_f32!("0", "7622503340244293", 0, false);
+ check_f32!("0", "30531780421733856", 0, false);
+ check_f32!("0", "21791061013936996", 0, false);
+ check_f32!("0", "09289376810193062", 0, false);
+ check_f32!("0", "03706067614257336", 0, false);
+ check_f32!("0", "028068351559340954", 0, false);
+ check_f32!("0", "012114629615098238", 0, false);
+ check_f32!("0", "004221370676532388", 0, false);
+ check_f32!("0", "002153817447833717", 0, false);
+ check_f32!("0", "0015924838953651488", 0, false);
+ check_f32!("0", "0008602388261351734", 0, false);
+ check_f32!("0", "00036393293703440577", 0, false);
+ check_f32!("0", "00013746770127909258", 0, false);
+ check_f32!("16407", "9462890625", 0, false);
+ check_f32!("1", "1754947011469036", -38, false);
+ check_f32!("7", "0064923216240854", -46, false);
+ check_f32!("8388614", "5", 0, false);
+ check_f32!("0", "", 2147483647, false);
+ check_f32!("4", "7019774032891500318749461488889827112746622270883500860350068251", -38, false);
+ check_f32!(
+ "3",
+ "\
+ 141592653589793238462643383279502884197169399375105820974944592307816406286\
+ 2089986280348253421170679",
+ 0,
+ false
+ );
+ check_f32!("2", "3509887016445750159374730744444913556373311135441750430175034126", -38, false);
+ check_f32!("1", "", 0, false);
+ check_f32!("7", "0060", -46, false);
+ check_f32!("7", "", -46, false);
+ check_f32!("3", "4028234664", 38, false);
+ check_f32!("3", "4028234665", 38, false);
+ check_f32!("3", "4028234666", 38, false);
+
+ check_f32!(
+ "0",
+ "\
+ 000000000000000000000000000000000000011754943508222875079687365372222456778186\
+ 655567720875215087517062784172594547271728515625",
+ 0,
+ false
+ );
+ check_f32!(
+ "0",
+ "\
+ 000000000000000000000000000000000000000000001401298464324817070923729583289916\
+ 13128026194187651577175706828388979108268586060148663818836212158203125",
+ 0,
+ false
+ );
+ check_f32!(
+ "0",
+ "\
+ 000000000000000000000000000000000000023509885615147285834557659820715330266457\
+ 17985517980855365926236850006129930346077117064851336181163787841796875",
+ 0,
+ false
+ );
+ check_f32!(
+ "0",
+ "\
+ 000000000000000000000000000000000000011754942106924410754870294448492873488270\
+ 52428745893333857174530571588870475618904265502351336181163787841796875",
+ 0,
+ false
+ );
+ check_f32!("1", "2345", 15, false);
+}