diff --git a/cpp/apidoc/Doxyfile b/cpp/apidoc/Doxyfile index de7777a6c1ca5..baf848ca237dd 100644 --- a/cpp/apidoc/Doxyfile +++ b/cpp/apidoc/Doxyfile @@ -1095,6 +1095,7 @@ EXCLUDE_PATTERNS = *-test.cc \ *test* \ *_generated.h \ *-benchmark.cc \ + *_codegen.py \ *internal* # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 9acd708343725..ce5d616bcb9ab 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -490,6 +490,8 @@ set(ARROW_UTIL_SRCS util/bitmap_builders.cc util/bitmap_ops.cc util/bpacking.cc + util/bpacking_scalar.cc + util/bpacking_simd_min.cc util/byte_size.cc util/byte_stream_split_internal.cc util/cancel.cc @@ -533,11 +535,8 @@ set(ARROW_UTIL_SRCS append_runtime_avx2_src(ARROW_UTIL_SRCS util/byte_stream_split_internal_avx2.cc) -append_runtime_avx2_src(ARROW_UTIL_SRCS util/bpacking_avx2.cc) -append_runtime_avx512_src(ARROW_UTIL_SRCS util/bpacking_avx512.cc) -if(ARROW_HAVE_NEON) - list(APPEND ARROW_UTIL_SRCS util/bpacking_neon.cc) -endif() +append_runtime_avx2_src(ARROW_UTIL_SRCS util/bpacking_simd_avx2.cc) +append_runtime_avx512_src(ARROW_UTIL_SRCS util/bpacking_simd_avx512.cc) if(ARROW_WITH_BROTLI) list(APPEND ARROW_UTIL_SRCS util/compression_brotli.cc) diff --git a/cpp/src/arrow/util/bit_stream_utils_internal.h b/cpp/src/arrow/util/bit_stream_utils_internal.h index 1f3b699e1ac6b..cccc2cb129528 100644 --- a/cpp/src/arrow/util/bit_stream_utils_internal.h +++ b/cpp/src/arrow/util/bit_stream_utils_internal.h @@ -123,6 +123,34 @@ inline uint64_t ReadLittleEndianWord(const uint8_t* buffer, int bytes_remaining) /// bytes in one read (e.g. encoded int). class BitReader { public: + template + struct UnpackFnDetect; + + template + struct UnpackFnDetect= sizeof(int))>> { + using type = internal::UnpackFn>; + }; + + template + struct UnpackFnDetect> { + using type = internal::UnpackFn; + }; + + /// The type for a function that can extract bit-packed integers. + template + using UnpackFn = typename UnpackFnDetect::type; + + /// Get the unack function most appropriated for this type and bit width. + template + static UnpackFn get_unpack_fn(int num_bits) { + // This is intimately linked to the GetBatch implementation + if constexpr (sizeof(T) >= sizeof(int)) { + return internal::get_unpack_fn>(num_bits); + } else { + return internal::get_unpack_fn(num_bits); + } + } + BitReader() noexcept = default; /// 'buffer' is the buffer to read from. The buffer's length is 'buffer_len'. @@ -148,6 +176,11 @@ class BitReader { template int GetBatch(int num_bits, T* v, int batch_size); + /// Get a number of values from the buffer. Return the number of values actually read. + /// @param unpack Function pointer to the unpack function for the correct bit width. + template + int GetBatch(int num_bits, T* v, int batch_size, UnpackFn unpack); + /// Reads a 'num_bytes'-sized value from the buffer and stores it in 'v'. T /// needs to be a little-endian native type and big enough to store /// 'num_bytes'. The value is assumed to be byte-aligned so the stream will @@ -297,7 +330,7 @@ inline bool BitReader::GetValue(int num_bits, T* v) { } template -inline int BitReader::GetBatch(int num_bits, T* v, int batch_size) { +int BitReader::GetBatch(int num_bits, T* v, int batch_size, UnpackFn unpack) { ARROW_DCHECK(buffer_ != NULL); ARROW_DCHECK_LE(num_bits, static_cast(sizeof(T) * 8)) << "num_bits: " << num_bits; @@ -323,19 +356,11 @@ inline int BitReader::GetBatch(int num_bits, T* v, int batch_size) { } } - if (sizeof(T) == 4) { + // unpack for uint16_t not as fast as unpack for uint32_t + memcpy. + if constexpr (sizeof(T) >= sizeof(32)) { int num_unpacked = - internal::unpack32(buffer + byte_offset, reinterpret_cast(v + i), - batch_size - i, num_bits); - i += num_unpacked; - byte_offset += num_unpacked * num_bits / 8; - } else if (sizeof(T) == 8 && num_bits > 32) { - // Use unpack64 only if num_bits is larger than 32 - // TODO (ARROW-13677): improve the performance of internal::unpack64 - // and remove the restriction of num_bits - int num_unpacked = - internal::unpack64(buffer + byte_offset, reinterpret_cast(v + i), - batch_size - i, num_bits); + unpack(buffer + byte_offset, reinterpret_cast*>(v + i), + batch_size - i); i += num_unpacked; byte_offset += num_unpacked * num_bits / 8; } else { @@ -345,8 +370,7 @@ inline int BitReader::GetBatch(int num_bits, T* v, int batch_size) { uint32_t unpack_buffer[buffer_size]; while (i < batch_size) { int unpack_size = std::min(buffer_size, batch_size - i); - int num_unpacked = - internal::unpack32(buffer + byte_offset, unpack_buffer, unpack_size, num_bits); + int num_unpacked = unpack(buffer + byte_offset, unpack_buffer, unpack_size); if (num_unpacked == 0) { break; } @@ -380,6 +404,11 @@ inline int BitReader::GetBatch(int num_bits, T* v, int batch_size) { return batch_size; } +template +inline int BitReader::GetBatch(int num_bits, T* v, int batch_size) { + return GetBatch(num_bits, v, batch_size, get_unpack_fn(num_bits)); +} + template inline bool BitReader::GetAligned(int num_bytes, T* v) { if (ARROW_PREDICT_FALSE(num_bytes > static_cast(sizeof(T)))) { diff --git a/cpp/src/arrow/util/bpacking.cc b/cpp/src/arrow/util/bpacking.cc index 990f76875aafd..4c3ec54a135a0 100644 --- a/cpp/src/arrow/util/bpacking.cc +++ b/cpp/src/arrow/util/bpacking.cc @@ -15,154 +15,76 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/util/bpacking_internal.h" +#include -#include "arrow/util/bpacking64_default_internal.h" -#include "arrow/util/bpacking_default_internal.h" -#include "arrow/util/cpu_info.h" +#include "arrow/util/bpacking_dispatch_internal.h" +#include "arrow/util/bpacking_internal.h" +#include "arrow/util/bpacking_scalar_internal.h" +#include "arrow/util/bpacking_simd_internal.h" #include "arrow/util/dispatch_internal.h" -#include "arrow/util/logging_internal.h" +namespace arrow::internal { + +namespace { + +template +struct UnpackDynamicFunction { + using FunctionType = decltype(&unpack_scalar); + using Implementation = std::pair; + + static auto implementations() { + return std::array { + // Current SIMD unpack algorithm works terribly on SSE4.2 due to lack of variable + // rhsift and poor xsimd fallback. + Implementation{DispatchLevel::NONE, &unpack_scalar}, #if defined(ARROW_HAVE_RUNTIME_AVX2) -# include "arrow/util/bpacking_avx2_internal.h" + Implementation{DispatchLevel::AVX2, &unpack_avx2}, #endif #if defined(ARROW_HAVE_RUNTIME_AVX512) -# include "arrow/util/bpacking_avx512_internal.h" + Implementation{DispatchLevel::AVX512, &unpack_avx512}, #endif -#if defined(ARROW_HAVE_NEON) -# include "arrow/util/bpacking_neon_internal.h" -#endif - -namespace arrow { -namespace internal { - -int unpack32_scalar(const uint8_t* in_, uint32_t* out, int batch_size, int num_bits) { - const uint32_t* in = reinterpret_cast(in_); + }; + } +}; - batch_size = batch_size / 32 * 32; - int num_loops = batch_size / 32; +} // namespace - switch (num_bits) { - case 0: - for (int i = 0; i < num_loops; ++i) in = nullunpacker32(in, out + i * 32); - break; - case 1: - for (int i = 0; i < num_loops; ++i) in = unpack1_32(in, out + i * 32); - break; - case 2: - for (int i = 0; i < num_loops; ++i) in = unpack2_32(in, out + i * 32); - break; - case 3: - for (int i = 0; i < num_loops; ++i) in = unpack3_32(in, out + i * 32); - break; - case 4: - for (int i = 0; i < num_loops; ++i) in = unpack4_32(in, out + i * 32); - break; - case 5: - for (int i = 0; i < num_loops; ++i) in = unpack5_32(in, out + i * 32); - break; - case 6: - for (int i = 0; i < num_loops; ++i) in = unpack6_32(in, out + i * 32); - break; - case 7: - for (int i = 0; i < num_loops; ++i) in = unpack7_32(in, out + i * 32); - break; - case 8: - for (int i = 0; i < num_loops; ++i) in = unpack8_32(in, out + i * 32); - break; - case 9: - for (int i = 0; i < num_loops; ++i) in = unpack9_32(in, out + i * 32); - break; - case 10: - for (int i = 0; i < num_loops; ++i) in = unpack10_32(in, out + i * 32); - break; - case 11: - for (int i = 0; i < num_loops; ++i) in = unpack11_32(in, out + i * 32); - break; - case 12: - for (int i = 0; i < num_loops; ++i) in = unpack12_32(in, out + i * 32); - break; - case 13: - for (int i = 0; i < num_loops; ++i) in = unpack13_32(in, out + i * 32); - break; - case 14: - for (int i = 0; i < num_loops; ++i) in = unpack14_32(in, out + i * 32); - break; - case 15: - for (int i = 0; i < num_loops; ++i) in = unpack15_32(in, out + i * 32); - break; - case 16: - for (int i = 0; i < num_loops; ++i) in = unpack16_32(in, out + i * 32); - break; - case 17: - for (int i = 0; i < num_loops; ++i) in = unpack17_32(in, out + i * 32); - break; - case 18: - for (int i = 0; i < num_loops; ++i) in = unpack18_32(in, out + i * 32); - break; - case 19: - for (int i = 0; i < num_loops; ++i) in = unpack19_32(in, out + i * 32); - break; - case 20: - for (int i = 0; i < num_loops; ++i) in = unpack20_32(in, out + i * 32); - break; - case 21: - for (int i = 0; i < num_loops; ++i) in = unpack21_32(in, out + i * 32); - break; - case 22: - for (int i = 0; i < num_loops; ++i) in = unpack22_32(in, out + i * 32); - break; - case 23: - for (int i = 0; i < num_loops; ++i) in = unpack23_32(in, out + i * 32); - break; - case 24: - for (int i = 0; i < num_loops; ++i) in = unpack24_32(in, out + i * 32); - break; - case 25: - for (int i = 0; i < num_loops; ++i) in = unpack25_32(in, out + i * 32); - break; - case 26: - for (int i = 0; i < num_loops; ++i) in = unpack26_32(in, out + i * 32); - break; - case 27: - for (int i = 0; i < num_loops; ++i) in = unpack27_32(in, out + i * 32); - break; - case 28: - for (int i = 0; i < num_loops; ++i) in = unpack28_32(in, out + i * 32); - break; - case 29: - for (int i = 0; i < num_loops; ++i) in = unpack29_32(in, out + i * 32); - break; - case 30: - for (int i = 0; i < num_loops; ++i) in = unpack30_32(in, out + i * 32); - break; - case 31: - for (int i = 0; i < num_loops; ++i) in = unpack31_32(in, out + i * 32); - break; - case 32: - for (int i = 0; i < num_loops; ++i) in = unpack32_32(in, out + i * 32); - break; - default: - DCHECK(false) << "Unsupported num_bits"; +template +int unpack(const uint8_t* in, Uint* out, int batch_size, int num_bits) { + if constexpr (std::is_same_v) { + // Current SIMD unpack function do not out beat scalar implementation for uin16_t + return unpack_scalar(in, out, batch_size, num_bits); + } else { +#if defined(ARROW_HAVE_NEON) + return unpack_neon(in, out, batch_size, num_bits); +#else + static DynamicDispatch > dispatch; + return dispatch.func(in, out, batch_size, num_bits); +#endif } - - return batch_size; } +template int unpack(const uint8_t*, uint16_t*, int, int); +template int unpack(const uint8_t*, uint32_t*, int, int); +template int unpack(const uint8_t*, uint64_t*, int, int); + namespace { -struct Unpack32DynamicFunction { - using FunctionType = decltype(&unpack32_scalar); +template +struct GetUnpackDynamicFunction { + using FunctionType = decltype(&get_unpack_fn_scalar); + using Implementation = std::pair; - static std::vector> implementations() { - return {{DispatchLevel::NONE, unpack32_scalar} + static auto implementations() { + return std::array { + // Current SIMD unpack algorithm works terribly on SSE4.2 due to lack of variable + // rhsift and poor xsimd fallback. + Implementation{DispatchLevel::NONE, &get_unpack_fn_scalar}, #if defined(ARROW_HAVE_RUNTIME_AVX2) - , - {DispatchLevel::AVX2, unpack32_avx2} + Implementation{DispatchLevel::AVX2, &get_unpack_fn_avx2}, #endif #if defined(ARROW_HAVE_RUNTIME_AVX512) - , - {DispatchLevel::AVX512, unpack32_avx512} + Implementation{DispatchLevel::AVX512, &get_unpack_fn_avx512}, #endif }; } @@ -170,226 +92,23 @@ struct Unpack32DynamicFunction { } // namespace -int unpack32(const uint8_t* in, uint32_t* out, int batch_size, int num_bits) { +template +UnpackFn get_unpack_fn(int num_bits) { + if constexpr (std::is_same_v) { + // Current SIMD unpack function do not out beat scalar implementation for uin16_t + return get_unpack_fn_scalar(num_bits); + } else { #if defined(ARROW_HAVE_NEON) - return unpack32_neon(in, out, batch_size, num_bits); + return get_unpack_fn_neon(num_bits); #else - static DynamicDispatch dispatch; - return dispatch.func(in, out, batch_size, num_bits); + static DynamicDispatch > dispatch; + return dispatch.func(num_bits); #endif -} - -int unpack64_scalar(const uint8_t* in, uint64_t* out, int batch_size, int num_bits) { - batch_size = batch_size / 32 * 32; - int num_loops = batch_size / 32; - - switch (num_bits) { - case 0: - for (int i = 0; i < num_loops; ++i) in = unpack0_64(in, out + i * 32); - break; - case 1: - for (int i = 0; i < num_loops; ++i) in = unpack1_64(in, out + i * 32); - break; - case 2: - for (int i = 0; i < num_loops; ++i) in = unpack2_64(in, out + i * 32); - break; - case 3: - for (int i = 0; i < num_loops; ++i) in = unpack3_64(in, out + i * 32); - break; - case 4: - for (int i = 0; i < num_loops; ++i) in = unpack4_64(in, out + i * 32); - break; - case 5: - for (int i = 0; i < num_loops; ++i) in = unpack5_64(in, out + i * 32); - break; - case 6: - for (int i = 0; i < num_loops; ++i) in = unpack6_64(in, out + i * 32); - break; - case 7: - for (int i = 0; i < num_loops; ++i) in = unpack7_64(in, out + i * 32); - break; - case 8: - for (int i = 0; i < num_loops; ++i) in = unpack8_64(in, out + i * 32); - break; - case 9: - for (int i = 0; i < num_loops; ++i) in = unpack9_64(in, out + i * 32); - break; - case 10: - for (int i = 0; i < num_loops; ++i) in = unpack10_64(in, out + i * 32); - break; - case 11: - for (int i = 0; i < num_loops; ++i) in = unpack11_64(in, out + i * 32); - break; - case 12: - for (int i = 0; i < num_loops; ++i) in = unpack12_64(in, out + i * 32); - break; - case 13: - for (int i = 0; i < num_loops; ++i) in = unpack13_64(in, out + i * 32); - break; - case 14: - for (int i = 0; i < num_loops; ++i) in = unpack14_64(in, out + i * 32); - break; - case 15: - for (int i = 0; i < num_loops; ++i) in = unpack15_64(in, out + i * 32); - break; - case 16: - for (int i = 0; i < num_loops; ++i) in = unpack16_64(in, out + i * 32); - break; - case 17: - for (int i = 0; i < num_loops; ++i) in = unpack17_64(in, out + i * 32); - break; - case 18: - for (int i = 0; i < num_loops; ++i) in = unpack18_64(in, out + i * 32); - break; - case 19: - for (int i = 0; i < num_loops; ++i) in = unpack19_64(in, out + i * 32); - break; - case 20: - for (int i = 0; i < num_loops; ++i) in = unpack20_64(in, out + i * 32); - break; - case 21: - for (int i = 0; i < num_loops; ++i) in = unpack21_64(in, out + i * 32); - break; - case 22: - for (int i = 0; i < num_loops; ++i) in = unpack22_64(in, out + i * 32); - break; - case 23: - for (int i = 0; i < num_loops; ++i) in = unpack23_64(in, out + i * 32); - break; - case 24: - for (int i = 0; i < num_loops; ++i) in = unpack24_64(in, out + i * 32); - break; - case 25: - for (int i = 0; i < num_loops; ++i) in = unpack25_64(in, out + i * 32); - break; - case 26: - for (int i = 0; i < num_loops; ++i) in = unpack26_64(in, out + i * 32); - break; - case 27: - for (int i = 0; i < num_loops; ++i) in = unpack27_64(in, out + i * 32); - break; - case 28: - for (int i = 0; i < num_loops; ++i) in = unpack28_64(in, out + i * 32); - break; - case 29: - for (int i = 0; i < num_loops; ++i) in = unpack29_64(in, out + i * 32); - break; - case 30: - for (int i = 0; i < num_loops; ++i) in = unpack30_64(in, out + i * 32); - break; - case 31: - for (int i = 0; i < num_loops; ++i) in = unpack31_64(in, out + i * 32); - break; - case 32: - for (int i = 0; i < num_loops; ++i) in = unpack32_64(in, out + i * 32); - break; - case 33: - for (int i = 0; i < num_loops; ++i) in = unpack33_64(in, out + i * 32); - break; - case 34: - for (int i = 0; i < num_loops; ++i) in = unpack34_64(in, out + i * 32); - break; - case 35: - for (int i = 0; i < num_loops; ++i) in = unpack35_64(in, out + i * 32); - break; - case 36: - for (int i = 0; i < num_loops; ++i) in = unpack36_64(in, out + i * 32); - break; - case 37: - for (int i = 0; i < num_loops; ++i) in = unpack37_64(in, out + i * 32); - break; - case 38: - for (int i = 0; i < num_loops; ++i) in = unpack38_64(in, out + i * 32); - break; - case 39: - for (int i = 0; i < num_loops; ++i) in = unpack39_64(in, out + i * 32); - break; - case 40: - for (int i = 0; i < num_loops; ++i) in = unpack40_64(in, out + i * 32); - break; - case 41: - for (int i = 0; i < num_loops; ++i) in = unpack41_64(in, out + i * 32); - break; - case 42: - for (int i = 0; i < num_loops; ++i) in = unpack42_64(in, out + i * 32); - break; - case 43: - for (int i = 0; i < num_loops; ++i) in = unpack43_64(in, out + i * 32); - break; - case 44: - for (int i = 0; i < num_loops; ++i) in = unpack44_64(in, out + i * 32); - break; - case 45: - for (int i = 0; i < num_loops; ++i) in = unpack45_64(in, out + i * 32); - break; - case 46: - for (int i = 0; i < num_loops; ++i) in = unpack46_64(in, out + i * 32); - break; - case 47: - for (int i = 0; i < num_loops; ++i) in = unpack47_64(in, out + i * 32); - break; - case 48: - for (int i = 0; i < num_loops; ++i) in = unpack48_64(in, out + i * 32); - break; - case 49: - for (int i = 0; i < num_loops; ++i) in = unpack49_64(in, out + i * 32); - break; - case 50: - for (int i = 0; i < num_loops; ++i) in = unpack50_64(in, out + i * 32); - break; - case 51: - for (int i = 0; i < num_loops; ++i) in = unpack51_64(in, out + i * 32); - break; - case 52: - for (int i = 0; i < num_loops; ++i) in = unpack52_64(in, out + i * 32); - break; - case 53: - for (int i = 0; i < num_loops; ++i) in = unpack53_64(in, out + i * 32); - break; - case 54: - for (int i = 0; i < num_loops; ++i) in = unpack54_64(in, out + i * 32); - break; - case 55: - for (int i = 0; i < num_loops; ++i) in = unpack55_64(in, out + i * 32); - break; - case 56: - for (int i = 0; i < num_loops; ++i) in = unpack56_64(in, out + i * 32); - break; - case 57: - for (int i = 0; i < num_loops; ++i) in = unpack57_64(in, out + i * 32); - break; - case 58: - for (int i = 0; i < num_loops; ++i) in = unpack58_64(in, out + i * 32); - break; - case 59: - for (int i = 0; i < num_loops; ++i) in = unpack59_64(in, out + i * 32); - break; - case 60: - for (int i = 0; i < num_loops; ++i) in = unpack60_64(in, out + i * 32); - break; - case 61: - for (int i = 0; i < num_loops; ++i) in = unpack61_64(in, out + i * 32); - break; - case 62: - for (int i = 0; i < num_loops; ++i) in = unpack62_64(in, out + i * 32); - break; - case 63: - for (int i = 0; i < num_loops; ++i) in = unpack63_64(in, out + i * 32); - break; - case 64: - for (int i = 0; i < num_loops; ++i) in = unpack64_64(in, out + i * 32); - break; - default: - DCHECK(false) << "Unsupported num_bits"; } - - return batch_size; } -int unpack64(const uint8_t* in, uint64_t* out, int batch_size, int num_bits) { - // TODO: unpack64_neon, unpack64_avx2 and unpack64_avx512 - return unpack64_scalar(in, out, batch_size, num_bits); -} +template UnpackFn get_unpack_fn(int); +template UnpackFn get_unpack_fn(int); +template UnpackFn get_unpack_fn(int); -} // namespace internal -} // namespace arrow +} // namespace arrow::internal diff --git a/cpp/src/arrow/util/bpacking64_codegen.py b/cpp/src/arrow/util/bpacking64_codegen.py deleted file mode 100644 index 22135fcbb2358..0000000000000 --- a/cpp/src/arrow/util/bpacking64_codegen.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/bin/python - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# This script is modified from its original version in GitHub. Original source: -# https://github.com/lemire/FrameOfReference/blob/146948b6058a976bc7767262ad3a2ce201486b93/scripts/turbopacking64.py - -# Usage: -# python bpacking64_codegen.py > bpacking64_default_internal.h - -def howmany(bit): - """ how many values are we going to pack? """ - return 32 - - -def howmanywords(bit): - return (howmany(bit) * bit + 63)//64 - - -def howmanybytes(bit): - return (howmany(bit) * bit + 7)//8 - - -print('''// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// This file was generated by script which is modified from its original version in GitHub. -// Original source: -// https://github.com/lemire/FrameOfReference/blob/master/scripts/turbopacking64.py -// The original copyright notice follows. - -// This code is released under the -// Apache License Version 2.0 http://www.apache.org/licenses/. -// (c) Daniel Lemire 2013 - -#pragma once - -#include "arrow/util/bit_util.h" -#include "arrow/util/ubsan.h" - -namespace arrow { -namespace internal { -''') - - -print("inline const uint8_t* unpack0_64(const uint8_t* in, uint64_t* out) {") -print(f" for(int k = 0; k < {howmany(0)} ; k += 1) {{") -print(" out[k] = 0;") -print(" }") -print(" return in;") -print("}") - -for bit in range(1, 65): - print("") - print(f"inline const uint8_t* unpack{bit}_64(const uint8_t* in, uint64_t* out) {{") - - if(bit < 64): - print(f" const uint64_t mask = {((1 << bit)-1)}ULL;") - maskstr = " & mask" - if (bit == 64): - maskstr = "" # no need - - for k in range(howmanywords(bit)-1): - print(f" uint64_t w{k} = util::SafeLoadAs(in);") - print(f" w{k} = arrow::BitUtil::FromLittleEndian(w{k});") - print(" in += 8;") - k = howmanywords(bit) - 1 - if (bit % 2 == 0): - print(f" uint64_t w{k} = util::SafeLoadAs(in);") - print(f" w{k} = arrow::BitUtil::FromLittleEndian(w{k});") - print(" in += 8;") - else: - print(f" uint64_t w{k} = util::SafeLoadAs(in);") - print(f" w{k} = arrow::BitUtil::FromLittleEndian(w{k});") - print(" in += 4;") - - for j in range(howmany(bit)): - firstword = j * bit // 64 - secondword = (j * bit + bit - 1)//64 - firstshift = (j*bit) % 64 - firstshiftstr = f" >> {firstshift}" - if(firstshift == 0): - firstshiftstr = "" # no need - if(firstword == secondword): - if(firstshift + bit == 64): - print(f" out[{j}] = w{firstword}{firstshiftstr};") - else: - print(f" out[{j}] = (w{firstword}{firstshiftstr}){maskstr};") - else: - secondshift = (64-firstshift) - print(f" out[{j}] = ((w{firstword}{firstshiftstr}) | " - f"(w{firstword+1} << {secondshift})){maskstr};") - print("") - print(" return in;") - print("}") - -print(''' -} // namespace internal -} // namespace arrow''') diff --git a/cpp/src/arrow/util/bpacking64_default_internal.h b/cpp/src/arrow/util/bpacking64_default_internal.h deleted file mode 100644 index 256cdda87e34f..0000000000000 --- a/cpp/src/arrow/util/bpacking64_default_internal.h +++ /dev/null @@ -1,5640 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// This file was generated by script which is modified from its original version in -// GitHub. Original source: -// https://github.com/lemire/FrameOfReference/blob/146948b6058a976bc7767262ad3a2ce201486b93/scripts/turbopacking64.py -// The original copyright notice follows. - -// This code is released under the -// Apache License Version 2.0 http://www.apache.org/licenses/. -// (c) Daniel Lemire 2013 - -#pragma once - -#include "arrow/util/endian.h" -#include "arrow/util/ubsan.h" - -namespace arrow::internal { - -inline const uint8_t* unpack0_64(const uint8_t* in, uint64_t* out) { - for (int k = 0; k < 32; k += 1) { - out[k] = 0; - } - return in; -} - -inline const uint8_t* unpack1_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 1ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 4; - out[0] = (w0)&mask; - out[1] = (w0 >> 1) & mask; - out[2] = (w0 >> 2) & mask; - out[3] = (w0 >> 3) & mask; - out[4] = (w0 >> 4) & mask; - out[5] = (w0 >> 5) & mask; - out[6] = (w0 >> 6) & mask; - out[7] = (w0 >> 7) & mask; - out[8] = (w0 >> 8) & mask; - out[9] = (w0 >> 9) & mask; - out[10] = (w0 >> 10) & mask; - out[11] = (w0 >> 11) & mask; - out[12] = (w0 >> 12) & mask; - out[13] = (w0 >> 13) & mask; - out[14] = (w0 >> 14) & mask; - out[15] = (w0 >> 15) & mask; - out[16] = (w0 >> 16) & mask; - out[17] = (w0 >> 17) & mask; - out[18] = (w0 >> 18) & mask; - out[19] = (w0 >> 19) & mask; - out[20] = (w0 >> 20) & mask; - out[21] = (w0 >> 21) & mask; - out[22] = (w0 >> 22) & mask; - out[23] = (w0 >> 23) & mask; - out[24] = (w0 >> 24) & mask; - out[25] = (w0 >> 25) & mask; - out[26] = (w0 >> 26) & mask; - out[27] = (w0 >> 27) & mask; - out[28] = (w0 >> 28) & mask; - out[29] = (w0 >> 29) & mask; - out[30] = (w0 >> 30) & mask; - out[31] = (w0 >> 31) & mask; - - return in; -} - -inline const uint8_t* unpack2_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 3ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - out[0] = (w0)&mask; - out[1] = (w0 >> 2) & mask; - out[2] = (w0 >> 4) & mask; - out[3] = (w0 >> 6) & mask; - out[4] = (w0 >> 8) & mask; - out[5] = (w0 >> 10) & mask; - out[6] = (w0 >> 12) & mask; - out[7] = (w0 >> 14) & mask; - out[8] = (w0 >> 16) & mask; - out[9] = (w0 >> 18) & mask; - out[10] = (w0 >> 20) & mask; - out[11] = (w0 >> 22) & mask; - out[12] = (w0 >> 24) & mask; - out[13] = (w0 >> 26) & mask; - out[14] = (w0 >> 28) & mask; - out[15] = (w0 >> 30) & mask; - out[16] = (w0 >> 32) & mask; - out[17] = (w0 >> 34) & mask; - out[18] = (w0 >> 36) & mask; - out[19] = (w0 >> 38) & mask; - out[20] = (w0 >> 40) & mask; - out[21] = (w0 >> 42) & mask; - out[22] = (w0 >> 44) & mask; - out[23] = (w0 >> 46) & mask; - out[24] = (w0 >> 48) & mask; - out[25] = (w0 >> 50) & mask; - out[26] = (w0 >> 52) & mask; - out[27] = (w0 >> 54) & mask; - out[28] = (w0 >> 56) & mask; - out[29] = (w0 >> 58) & mask; - out[30] = (w0 >> 60) & mask; - out[31] = w0 >> 62; - - return in; -} - -inline const uint8_t* unpack3_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 7ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 4; - out[0] = (w0)&mask; - out[1] = (w0 >> 3) & mask; - out[2] = (w0 >> 6) & mask; - out[3] = (w0 >> 9) & mask; - out[4] = (w0 >> 12) & mask; - out[5] = (w0 >> 15) & mask; - out[6] = (w0 >> 18) & mask; - out[7] = (w0 >> 21) & mask; - out[8] = (w0 >> 24) & mask; - out[9] = (w0 >> 27) & mask; - out[10] = (w0 >> 30) & mask; - out[11] = (w0 >> 33) & mask; - out[12] = (w0 >> 36) & mask; - out[13] = (w0 >> 39) & mask; - out[14] = (w0 >> 42) & mask; - out[15] = (w0 >> 45) & mask; - out[16] = (w0 >> 48) & mask; - out[17] = (w0 >> 51) & mask; - out[18] = (w0 >> 54) & mask; - out[19] = (w0 >> 57) & mask; - out[20] = (w0 >> 60) & mask; - out[21] = ((w0 >> 63) | (w1 << 1)) & mask; - out[22] = (w1 >> 2) & mask; - out[23] = (w1 >> 5) & mask; - out[24] = (w1 >> 8) & mask; - out[25] = (w1 >> 11) & mask; - out[26] = (w1 >> 14) & mask; - out[27] = (w1 >> 17) & mask; - out[28] = (w1 >> 20) & mask; - out[29] = (w1 >> 23) & mask; - out[30] = (w1 >> 26) & mask; - out[31] = (w1 >> 29) & mask; - - return in; -} - -inline const uint8_t* unpack4_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 15ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - out[0] = (w0)&mask; - out[1] = (w0 >> 4) & mask; - out[2] = (w0 >> 8) & mask; - out[3] = (w0 >> 12) & mask; - out[4] = (w0 >> 16) & mask; - out[5] = (w0 >> 20) & mask; - out[6] = (w0 >> 24) & mask; - out[7] = (w0 >> 28) & mask; - out[8] = (w0 >> 32) & mask; - out[9] = (w0 >> 36) & mask; - out[10] = (w0 >> 40) & mask; - out[11] = (w0 >> 44) & mask; - out[12] = (w0 >> 48) & mask; - out[13] = (w0 >> 52) & mask; - out[14] = (w0 >> 56) & mask; - out[15] = w0 >> 60; - out[16] = (w1)&mask; - out[17] = (w1 >> 4) & mask; - out[18] = (w1 >> 8) & mask; - out[19] = (w1 >> 12) & mask; - out[20] = (w1 >> 16) & mask; - out[21] = (w1 >> 20) & mask; - out[22] = (w1 >> 24) & mask; - out[23] = (w1 >> 28) & mask; - out[24] = (w1 >> 32) & mask; - out[25] = (w1 >> 36) & mask; - out[26] = (w1 >> 40) & mask; - out[27] = (w1 >> 44) & mask; - out[28] = (w1 >> 48) & mask; - out[29] = (w1 >> 52) & mask; - out[30] = (w1 >> 56) & mask; - out[31] = w1 >> 60; - - return in; -} - -inline const uint8_t* unpack5_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 31ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 4; - out[0] = (w0)&mask; - out[1] = (w0 >> 5) & mask; - out[2] = (w0 >> 10) & mask; - out[3] = (w0 >> 15) & mask; - out[4] = (w0 >> 20) & mask; - out[5] = (w0 >> 25) & mask; - out[6] = (w0 >> 30) & mask; - out[7] = (w0 >> 35) & mask; - out[8] = (w0 >> 40) & mask; - out[9] = (w0 >> 45) & mask; - out[10] = (w0 >> 50) & mask; - out[11] = (w0 >> 55) & mask; - out[12] = ((w0 >> 60) | (w1 << 4)) & mask; - out[13] = (w1 >> 1) & mask; - out[14] = (w1 >> 6) & mask; - out[15] = (w1 >> 11) & mask; - out[16] = (w1 >> 16) & mask; - out[17] = (w1 >> 21) & mask; - out[18] = (w1 >> 26) & mask; - out[19] = (w1 >> 31) & mask; - out[20] = (w1 >> 36) & mask; - out[21] = (w1 >> 41) & mask; - out[22] = (w1 >> 46) & mask; - out[23] = (w1 >> 51) & mask; - out[24] = (w1 >> 56) & mask; - out[25] = ((w1 >> 61) | (w2 << 3)) & mask; - out[26] = (w2 >> 2) & mask; - out[27] = (w2 >> 7) & mask; - out[28] = (w2 >> 12) & mask; - out[29] = (w2 >> 17) & mask; - out[30] = (w2 >> 22) & mask; - out[31] = (w2 >> 27) & mask; - - return in; -} - -inline const uint8_t* unpack6_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 63ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - out[0] = (w0)&mask; - out[1] = (w0 >> 6) & mask; - out[2] = (w0 >> 12) & mask; - out[3] = (w0 >> 18) & mask; - out[4] = (w0 >> 24) & mask; - out[5] = (w0 >> 30) & mask; - out[6] = (w0 >> 36) & mask; - out[7] = (w0 >> 42) & mask; - out[8] = (w0 >> 48) & mask; - out[9] = (w0 >> 54) & mask; - out[10] = ((w0 >> 60) | (w1 << 4)) & mask; - out[11] = (w1 >> 2) & mask; - out[12] = (w1 >> 8) & mask; - out[13] = (w1 >> 14) & mask; - out[14] = (w1 >> 20) & mask; - out[15] = (w1 >> 26) & mask; - out[16] = (w1 >> 32) & mask; - out[17] = (w1 >> 38) & mask; - out[18] = (w1 >> 44) & mask; - out[19] = (w1 >> 50) & mask; - out[20] = (w1 >> 56) & mask; - out[21] = ((w1 >> 62) | (w2 << 2)) & mask; - out[22] = (w2 >> 4) & mask; - out[23] = (w2 >> 10) & mask; - out[24] = (w2 >> 16) & mask; - out[25] = (w2 >> 22) & mask; - out[26] = (w2 >> 28) & mask; - out[27] = (w2 >> 34) & mask; - out[28] = (w2 >> 40) & mask; - out[29] = (w2 >> 46) & mask; - out[30] = (w2 >> 52) & mask; - out[31] = w2 >> 58; - - return in; -} - -inline const uint8_t* unpack7_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 127ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 4; - out[0] = (w0)&mask; - out[1] = (w0 >> 7) & mask; - out[2] = (w0 >> 14) & mask; - out[3] = (w0 >> 21) & mask; - out[4] = (w0 >> 28) & mask; - out[5] = (w0 >> 35) & mask; - out[6] = (w0 >> 42) & mask; - out[7] = (w0 >> 49) & mask; - out[8] = (w0 >> 56) & mask; - out[9] = ((w0 >> 63) | (w1 << 1)) & mask; - out[10] = (w1 >> 6) & mask; - out[11] = (w1 >> 13) & mask; - out[12] = (w1 >> 20) & mask; - out[13] = (w1 >> 27) & mask; - out[14] = (w1 >> 34) & mask; - out[15] = (w1 >> 41) & mask; - out[16] = (w1 >> 48) & mask; - out[17] = (w1 >> 55) & mask; - out[18] = ((w1 >> 62) | (w2 << 2)) & mask; - out[19] = (w2 >> 5) & mask; - out[20] = (w2 >> 12) & mask; - out[21] = (w2 >> 19) & mask; - out[22] = (w2 >> 26) & mask; - out[23] = (w2 >> 33) & mask; - out[24] = (w2 >> 40) & mask; - out[25] = (w2 >> 47) & mask; - out[26] = (w2 >> 54) & mask; - out[27] = ((w2 >> 61) | (w3 << 3)) & mask; - out[28] = (w3 >> 4) & mask; - out[29] = (w3 >> 11) & mask; - out[30] = (w3 >> 18) & mask; - out[31] = (w3 >> 25) & mask; - - return in; -} - -inline const uint8_t* unpack8_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 255ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - out[0] = (w0)&mask; - out[1] = (w0 >> 8) & mask; - out[2] = (w0 >> 16) & mask; - out[3] = (w0 >> 24) & mask; - out[4] = (w0 >> 32) & mask; - out[5] = (w0 >> 40) & mask; - out[6] = (w0 >> 48) & mask; - out[7] = w0 >> 56; - out[8] = (w1)&mask; - out[9] = (w1 >> 8) & mask; - out[10] = (w1 >> 16) & mask; - out[11] = (w1 >> 24) & mask; - out[12] = (w1 >> 32) & mask; - out[13] = (w1 >> 40) & mask; - out[14] = (w1 >> 48) & mask; - out[15] = w1 >> 56; - out[16] = (w2)&mask; - out[17] = (w2 >> 8) & mask; - out[18] = (w2 >> 16) & mask; - out[19] = (w2 >> 24) & mask; - out[20] = (w2 >> 32) & mask; - out[21] = (w2 >> 40) & mask; - out[22] = (w2 >> 48) & mask; - out[23] = w2 >> 56; - out[24] = (w3)&mask; - out[25] = (w3 >> 8) & mask; - out[26] = (w3 >> 16) & mask; - out[27] = (w3 >> 24) & mask; - out[28] = (w3 >> 32) & mask; - out[29] = (w3 >> 40) & mask; - out[30] = (w3 >> 48) & mask; - out[31] = w3 >> 56; - - return in; -} - -inline const uint8_t* unpack9_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 511ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 4; - out[0] = (w0)&mask; - out[1] = (w0 >> 9) & mask; - out[2] = (w0 >> 18) & mask; - out[3] = (w0 >> 27) & mask; - out[4] = (w0 >> 36) & mask; - out[5] = (w0 >> 45) & mask; - out[6] = (w0 >> 54) & mask; - out[7] = ((w0 >> 63) | (w1 << 1)) & mask; - out[8] = (w1 >> 8) & mask; - out[9] = (w1 >> 17) & mask; - out[10] = (w1 >> 26) & mask; - out[11] = (w1 >> 35) & mask; - out[12] = (w1 >> 44) & mask; - out[13] = (w1 >> 53) & mask; - out[14] = ((w1 >> 62) | (w2 << 2)) & mask; - out[15] = (w2 >> 7) & mask; - out[16] = (w2 >> 16) & mask; - out[17] = (w2 >> 25) & mask; - out[18] = (w2 >> 34) & mask; - out[19] = (w2 >> 43) & mask; - out[20] = (w2 >> 52) & mask; - out[21] = ((w2 >> 61) | (w3 << 3)) & mask; - out[22] = (w3 >> 6) & mask; - out[23] = (w3 >> 15) & mask; - out[24] = (w3 >> 24) & mask; - out[25] = (w3 >> 33) & mask; - out[26] = (w3 >> 42) & mask; - out[27] = (w3 >> 51) & mask; - out[28] = ((w3 >> 60) | (w4 << 4)) & mask; - out[29] = (w4 >> 5) & mask; - out[30] = (w4 >> 14) & mask; - out[31] = (w4 >> 23) & mask; - - return in; -} - -inline const uint8_t* unpack10_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 1023ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - out[0] = (w0)&mask; - out[1] = (w0 >> 10) & mask; - out[2] = (w0 >> 20) & mask; - out[3] = (w0 >> 30) & mask; - out[4] = (w0 >> 40) & mask; - out[5] = (w0 >> 50) & mask; - out[6] = ((w0 >> 60) | (w1 << 4)) & mask; - out[7] = (w1 >> 6) & mask; - out[8] = (w1 >> 16) & mask; - out[9] = (w1 >> 26) & mask; - out[10] = (w1 >> 36) & mask; - out[11] = (w1 >> 46) & mask; - out[12] = ((w1 >> 56) | (w2 << 8)) & mask; - out[13] = (w2 >> 2) & mask; - out[14] = (w2 >> 12) & mask; - out[15] = (w2 >> 22) & mask; - out[16] = (w2 >> 32) & mask; - out[17] = (w2 >> 42) & mask; - out[18] = (w2 >> 52) & mask; - out[19] = ((w2 >> 62) | (w3 << 2)) & mask; - out[20] = (w3 >> 8) & mask; - out[21] = (w3 >> 18) & mask; - out[22] = (w3 >> 28) & mask; - out[23] = (w3 >> 38) & mask; - out[24] = (w3 >> 48) & mask; - out[25] = ((w3 >> 58) | (w4 << 6)) & mask; - out[26] = (w4 >> 4) & mask; - out[27] = (w4 >> 14) & mask; - out[28] = (w4 >> 24) & mask; - out[29] = (w4 >> 34) & mask; - out[30] = (w4 >> 44) & mask; - out[31] = w4 >> 54; - - return in; -} - -inline const uint8_t* unpack11_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 2047ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 4; - out[0] = (w0)&mask; - out[1] = (w0 >> 11) & mask; - out[2] = (w0 >> 22) & mask; - out[3] = (w0 >> 33) & mask; - out[4] = (w0 >> 44) & mask; - out[5] = ((w0 >> 55) | (w1 << 9)) & mask; - out[6] = (w1 >> 2) & mask; - out[7] = (w1 >> 13) & mask; - out[8] = (w1 >> 24) & mask; - out[9] = (w1 >> 35) & mask; - out[10] = (w1 >> 46) & mask; - out[11] = ((w1 >> 57) | (w2 << 7)) & mask; - out[12] = (w2 >> 4) & mask; - out[13] = (w2 >> 15) & mask; - out[14] = (w2 >> 26) & mask; - out[15] = (w2 >> 37) & mask; - out[16] = (w2 >> 48) & mask; - out[17] = ((w2 >> 59) | (w3 << 5)) & mask; - out[18] = (w3 >> 6) & mask; - out[19] = (w3 >> 17) & mask; - out[20] = (w3 >> 28) & mask; - out[21] = (w3 >> 39) & mask; - out[22] = (w3 >> 50) & mask; - out[23] = ((w3 >> 61) | (w4 << 3)) & mask; - out[24] = (w4 >> 8) & mask; - out[25] = (w4 >> 19) & mask; - out[26] = (w4 >> 30) & mask; - out[27] = (w4 >> 41) & mask; - out[28] = (w4 >> 52) & mask; - out[29] = ((w4 >> 63) | (w5 << 1)) & mask; - out[30] = (w5 >> 10) & mask; - out[31] = (w5 >> 21) & mask; - - return in; -} - -inline const uint8_t* unpack12_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 4095ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - out[0] = (w0)&mask; - out[1] = (w0 >> 12) & mask; - out[2] = (w0 >> 24) & mask; - out[3] = (w0 >> 36) & mask; - out[4] = (w0 >> 48) & mask; - out[5] = ((w0 >> 60) | (w1 << 4)) & mask; - out[6] = (w1 >> 8) & mask; - out[7] = (w1 >> 20) & mask; - out[8] = (w1 >> 32) & mask; - out[9] = (w1 >> 44) & mask; - out[10] = ((w1 >> 56) | (w2 << 8)) & mask; - out[11] = (w2 >> 4) & mask; - out[12] = (w2 >> 16) & mask; - out[13] = (w2 >> 28) & mask; - out[14] = (w2 >> 40) & mask; - out[15] = w2 >> 52; - out[16] = (w3)&mask; - out[17] = (w3 >> 12) & mask; - out[18] = (w3 >> 24) & mask; - out[19] = (w3 >> 36) & mask; - out[20] = (w3 >> 48) & mask; - out[21] = ((w3 >> 60) | (w4 << 4)) & mask; - out[22] = (w4 >> 8) & mask; - out[23] = (w4 >> 20) & mask; - out[24] = (w4 >> 32) & mask; - out[25] = (w4 >> 44) & mask; - out[26] = ((w4 >> 56) | (w5 << 8)) & mask; - out[27] = (w5 >> 4) & mask; - out[28] = (w5 >> 16) & mask; - out[29] = (w5 >> 28) & mask; - out[30] = (w5 >> 40) & mask; - out[31] = w5 >> 52; - - return in; -} - -inline const uint8_t* unpack13_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 8191ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 4; - out[0] = (w0)&mask; - out[1] = (w0 >> 13) & mask; - out[2] = (w0 >> 26) & mask; - out[3] = (w0 >> 39) & mask; - out[4] = ((w0 >> 52) | (w1 << 12)) & mask; - out[5] = (w1 >> 1) & mask; - out[6] = (w1 >> 14) & mask; - out[7] = (w1 >> 27) & mask; - out[8] = (w1 >> 40) & mask; - out[9] = ((w1 >> 53) | (w2 << 11)) & mask; - out[10] = (w2 >> 2) & mask; - out[11] = (w2 >> 15) & mask; - out[12] = (w2 >> 28) & mask; - out[13] = (w2 >> 41) & mask; - out[14] = ((w2 >> 54) | (w3 << 10)) & mask; - out[15] = (w3 >> 3) & mask; - out[16] = (w3 >> 16) & mask; - out[17] = (w3 >> 29) & mask; - out[18] = (w3 >> 42) & mask; - out[19] = ((w3 >> 55) | (w4 << 9)) & mask; - out[20] = (w4 >> 4) & mask; - out[21] = (w4 >> 17) & mask; - out[22] = (w4 >> 30) & mask; - out[23] = (w4 >> 43) & mask; - out[24] = ((w4 >> 56) | (w5 << 8)) & mask; - out[25] = (w5 >> 5) & mask; - out[26] = (w5 >> 18) & mask; - out[27] = (w5 >> 31) & mask; - out[28] = (w5 >> 44) & mask; - out[29] = ((w5 >> 57) | (w6 << 7)) & mask; - out[30] = (w6 >> 6) & mask; - out[31] = (w6 >> 19) & mask; - - return in; -} - -inline const uint8_t* unpack14_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 16383ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - out[0] = (w0)&mask; - out[1] = (w0 >> 14) & mask; - out[2] = (w0 >> 28) & mask; - out[3] = (w0 >> 42) & mask; - out[4] = ((w0 >> 56) | (w1 << 8)) & mask; - out[5] = (w1 >> 6) & mask; - out[6] = (w1 >> 20) & mask; - out[7] = (w1 >> 34) & mask; - out[8] = (w1 >> 48) & mask; - out[9] = ((w1 >> 62) | (w2 << 2)) & mask; - out[10] = (w2 >> 12) & mask; - out[11] = (w2 >> 26) & mask; - out[12] = (w2 >> 40) & mask; - out[13] = ((w2 >> 54) | (w3 << 10)) & mask; - out[14] = (w3 >> 4) & mask; - out[15] = (w3 >> 18) & mask; - out[16] = (w3 >> 32) & mask; - out[17] = (w3 >> 46) & mask; - out[18] = ((w3 >> 60) | (w4 << 4)) & mask; - out[19] = (w4 >> 10) & mask; - out[20] = (w4 >> 24) & mask; - out[21] = (w4 >> 38) & mask; - out[22] = ((w4 >> 52) | (w5 << 12)) & mask; - out[23] = (w5 >> 2) & mask; - out[24] = (w5 >> 16) & mask; - out[25] = (w5 >> 30) & mask; - out[26] = (w5 >> 44) & mask; - out[27] = ((w5 >> 58) | (w6 << 6)) & mask; - out[28] = (w6 >> 8) & mask; - out[29] = (w6 >> 22) & mask; - out[30] = (w6 >> 36) & mask; - out[31] = w6 >> 50; - - return in; -} - -inline const uint8_t* unpack15_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 32767ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 4; - out[0] = (w0)&mask; - out[1] = (w0 >> 15) & mask; - out[2] = (w0 >> 30) & mask; - out[3] = (w0 >> 45) & mask; - out[4] = ((w0 >> 60) | (w1 << 4)) & mask; - out[5] = (w1 >> 11) & mask; - out[6] = (w1 >> 26) & mask; - out[7] = (w1 >> 41) & mask; - out[8] = ((w1 >> 56) | (w2 << 8)) & mask; - out[9] = (w2 >> 7) & mask; - out[10] = (w2 >> 22) & mask; - out[11] = (w2 >> 37) & mask; - out[12] = ((w2 >> 52) | (w3 << 12)) & mask; - out[13] = (w3 >> 3) & mask; - out[14] = (w3 >> 18) & mask; - out[15] = (w3 >> 33) & mask; - out[16] = (w3 >> 48) & mask; - out[17] = ((w3 >> 63) | (w4 << 1)) & mask; - out[18] = (w4 >> 14) & mask; - out[19] = (w4 >> 29) & mask; - out[20] = (w4 >> 44) & mask; - out[21] = ((w4 >> 59) | (w5 << 5)) & mask; - out[22] = (w5 >> 10) & mask; - out[23] = (w5 >> 25) & mask; - out[24] = (w5 >> 40) & mask; - out[25] = ((w5 >> 55) | (w6 << 9)) & mask; - out[26] = (w6 >> 6) & mask; - out[27] = (w6 >> 21) & mask; - out[28] = (w6 >> 36) & mask; - out[29] = ((w6 >> 51) | (w7 << 13)) & mask; - out[30] = (w7 >> 2) & mask; - out[31] = (w7 >> 17) & mask; - - return in; -} - -inline const uint8_t* unpack16_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 65535ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - out[0] = (w0)&mask; - out[1] = (w0 >> 16) & mask; - out[2] = (w0 >> 32) & mask; - out[3] = w0 >> 48; - out[4] = (w1)&mask; - out[5] = (w1 >> 16) & mask; - out[6] = (w1 >> 32) & mask; - out[7] = w1 >> 48; - out[8] = (w2)&mask; - out[9] = (w2 >> 16) & mask; - out[10] = (w2 >> 32) & mask; - out[11] = w2 >> 48; - out[12] = (w3)&mask; - out[13] = (w3 >> 16) & mask; - out[14] = (w3 >> 32) & mask; - out[15] = w3 >> 48; - out[16] = (w4)&mask; - out[17] = (w4 >> 16) & mask; - out[18] = (w4 >> 32) & mask; - out[19] = w4 >> 48; - out[20] = (w5)&mask; - out[21] = (w5 >> 16) & mask; - out[22] = (w5 >> 32) & mask; - out[23] = w5 >> 48; - out[24] = (w6)&mask; - out[25] = (w6 >> 16) & mask; - out[26] = (w6 >> 32) & mask; - out[27] = w6 >> 48; - out[28] = (w7)&mask; - out[29] = (w7 >> 16) & mask; - out[30] = (w7 >> 32) & mask; - out[31] = w7 >> 48; - - return in; -} - -inline const uint8_t* unpack17_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 131071ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 4; - out[0] = (w0)&mask; - out[1] = (w0 >> 17) & mask; - out[2] = (w0 >> 34) & mask; - out[3] = ((w0 >> 51) | (w1 << 13)) & mask; - out[4] = (w1 >> 4) & mask; - out[5] = (w1 >> 21) & mask; - out[6] = (w1 >> 38) & mask; - out[7] = ((w1 >> 55) | (w2 << 9)) & mask; - out[8] = (w2 >> 8) & mask; - out[9] = (w2 >> 25) & mask; - out[10] = (w2 >> 42) & mask; - out[11] = ((w2 >> 59) | (w3 << 5)) & mask; - out[12] = (w3 >> 12) & mask; - out[13] = (w3 >> 29) & mask; - out[14] = (w3 >> 46) & mask; - out[15] = ((w3 >> 63) | (w4 << 1)) & mask; - out[16] = (w4 >> 16) & mask; - out[17] = (w4 >> 33) & mask; - out[18] = ((w4 >> 50) | (w5 << 14)) & mask; - out[19] = (w5 >> 3) & mask; - out[20] = (w5 >> 20) & mask; - out[21] = (w5 >> 37) & mask; - out[22] = ((w5 >> 54) | (w6 << 10)) & mask; - out[23] = (w6 >> 7) & mask; - out[24] = (w6 >> 24) & mask; - out[25] = (w6 >> 41) & mask; - out[26] = ((w6 >> 58) | (w7 << 6)) & mask; - out[27] = (w7 >> 11) & mask; - out[28] = (w7 >> 28) & mask; - out[29] = (w7 >> 45) & mask; - out[30] = ((w7 >> 62) | (w8 << 2)) & mask; - out[31] = (w8 >> 15) & mask; - - return in; -} - -inline const uint8_t* unpack18_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 262143ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - out[0] = (w0)&mask; - out[1] = (w0 >> 18) & mask; - out[2] = (w0 >> 36) & mask; - out[3] = ((w0 >> 54) | (w1 << 10)) & mask; - out[4] = (w1 >> 8) & mask; - out[5] = (w1 >> 26) & mask; - out[6] = (w1 >> 44) & mask; - out[7] = ((w1 >> 62) | (w2 << 2)) & mask; - out[8] = (w2 >> 16) & mask; - out[9] = (w2 >> 34) & mask; - out[10] = ((w2 >> 52) | (w3 << 12)) & mask; - out[11] = (w3 >> 6) & mask; - out[12] = (w3 >> 24) & mask; - out[13] = (w3 >> 42) & mask; - out[14] = ((w3 >> 60) | (w4 << 4)) & mask; - out[15] = (w4 >> 14) & mask; - out[16] = (w4 >> 32) & mask; - out[17] = ((w4 >> 50) | (w5 << 14)) & mask; - out[18] = (w5 >> 4) & mask; - out[19] = (w5 >> 22) & mask; - out[20] = (w5 >> 40) & mask; - out[21] = ((w5 >> 58) | (w6 << 6)) & mask; - out[22] = (w6 >> 12) & mask; - out[23] = (w6 >> 30) & mask; - out[24] = ((w6 >> 48) | (w7 << 16)) & mask; - out[25] = (w7 >> 2) & mask; - out[26] = (w7 >> 20) & mask; - out[27] = (w7 >> 38) & mask; - out[28] = ((w7 >> 56) | (w8 << 8)) & mask; - out[29] = (w8 >> 10) & mask; - out[30] = (w8 >> 28) & mask; - out[31] = w8 >> 46; - - return in; -} - -inline const uint8_t* unpack19_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 524287ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 4; - out[0] = (w0)&mask; - out[1] = (w0 >> 19) & mask; - out[2] = (w0 >> 38) & mask; - out[3] = ((w0 >> 57) | (w1 << 7)) & mask; - out[4] = (w1 >> 12) & mask; - out[5] = (w1 >> 31) & mask; - out[6] = ((w1 >> 50) | (w2 << 14)) & mask; - out[7] = (w2 >> 5) & mask; - out[8] = (w2 >> 24) & mask; - out[9] = (w2 >> 43) & mask; - out[10] = ((w2 >> 62) | (w3 << 2)) & mask; - out[11] = (w3 >> 17) & mask; - out[12] = (w3 >> 36) & mask; - out[13] = ((w3 >> 55) | (w4 << 9)) & mask; - out[14] = (w4 >> 10) & mask; - out[15] = (w4 >> 29) & mask; - out[16] = ((w4 >> 48) | (w5 << 16)) & mask; - out[17] = (w5 >> 3) & mask; - out[18] = (w5 >> 22) & mask; - out[19] = (w5 >> 41) & mask; - out[20] = ((w5 >> 60) | (w6 << 4)) & mask; - out[21] = (w6 >> 15) & mask; - out[22] = (w6 >> 34) & mask; - out[23] = ((w6 >> 53) | (w7 << 11)) & mask; - out[24] = (w7 >> 8) & mask; - out[25] = (w7 >> 27) & mask; - out[26] = ((w7 >> 46) | (w8 << 18)) & mask; - out[27] = (w8 >> 1) & mask; - out[28] = (w8 >> 20) & mask; - out[29] = (w8 >> 39) & mask; - out[30] = ((w8 >> 58) | (w9 << 6)) & mask; - out[31] = (w9 >> 13) & mask; - - return in; -} - -inline const uint8_t* unpack20_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 1048575ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - out[0] = (w0)&mask; - out[1] = (w0 >> 20) & mask; - out[2] = (w0 >> 40) & mask; - out[3] = ((w0 >> 60) | (w1 << 4)) & mask; - out[4] = (w1 >> 16) & mask; - out[5] = (w1 >> 36) & mask; - out[6] = ((w1 >> 56) | (w2 << 8)) & mask; - out[7] = (w2 >> 12) & mask; - out[8] = (w2 >> 32) & mask; - out[9] = ((w2 >> 52) | (w3 << 12)) & mask; - out[10] = (w3 >> 8) & mask; - out[11] = (w3 >> 28) & mask; - out[12] = ((w3 >> 48) | (w4 << 16)) & mask; - out[13] = (w4 >> 4) & mask; - out[14] = (w4 >> 24) & mask; - out[15] = w4 >> 44; - out[16] = (w5)&mask; - out[17] = (w5 >> 20) & mask; - out[18] = (w5 >> 40) & mask; - out[19] = ((w5 >> 60) | (w6 << 4)) & mask; - out[20] = (w6 >> 16) & mask; - out[21] = (w6 >> 36) & mask; - out[22] = ((w6 >> 56) | (w7 << 8)) & mask; - out[23] = (w7 >> 12) & mask; - out[24] = (w7 >> 32) & mask; - out[25] = ((w7 >> 52) | (w8 << 12)) & mask; - out[26] = (w8 >> 8) & mask; - out[27] = (w8 >> 28) & mask; - out[28] = ((w8 >> 48) | (w9 << 16)) & mask; - out[29] = (w9 >> 4) & mask; - out[30] = (w9 >> 24) & mask; - out[31] = w9 >> 44; - - return in; -} - -inline const uint8_t* unpack21_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 2097151ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 4; - out[0] = (w0)&mask; - out[1] = (w0 >> 21) & mask; - out[2] = (w0 >> 42) & mask; - out[3] = ((w0 >> 63) | (w1 << 1)) & mask; - out[4] = (w1 >> 20) & mask; - out[5] = (w1 >> 41) & mask; - out[6] = ((w1 >> 62) | (w2 << 2)) & mask; - out[7] = (w2 >> 19) & mask; - out[8] = (w2 >> 40) & mask; - out[9] = ((w2 >> 61) | (w3 << 3)) & mask; - out[10] = (w3 >> 18) & mask; - out[11] = (w3 >> 39) & mask; - out[12] = ((w3 >> 60) | (w4 << 4)) & mask; - out[13] = (w4 >> 17) & mask; - out[14] = (w4 >> 38) & mask; - out[15] = ((w4 >> 59) | (w5 << 5)) & mask; - out[16] = (w5 >> 16) & mask; - out[17] = (w5 >> 37) & mask; - out[18] = ((w5 >> 58) | (w6 << 6)) & mask; - out[19] = (w6 >> 15) & mask; - out[20] = (w6 >> 36) & mask; - out[21] = ((w6 >> 57) | (w7 << 7)) & mask; - out[22] = (w7 >> 14) & mask; - out[23] = (w7 >> 35) & mask; - out[24] = ((w7 >> 56) | (w8 << 8)) & mask; - out[25] = (w8 >> 13) & mask; - out[26] = (w8 >> 34) & mask; - out[27] = ((w8 >> 55) | (w9 << 9)) & mask; - out[28] = (w9 >> 12) & mask; - out[29] = (w9 >> 33) & mask; - out[30] = ((w9 >> 54) | (w10 << 10)) & mask; - out[31] = (w10 >> 11) & mask; - - return in; -} - -inline const uint8_t* unpack22_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 4194303ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - out[0] = (w0)&mask; - out[1] = (w0 >> 22) & mask; - out[2] = ((w0 >> 44) | (w1 << 20)) & mask; - out[3] = (w1 >> 2) & mask; - out[4] = (w1 >> 24) & mask; - out[5] = ((w1 >> 46) | (w2 << 18)) & mask; - out[6] = (w2 >> 4) & mask; - out[7] = (w2 >> 26) & mask; - out[8] = ((w2 >> 48) | (w3 << 16)) & mask; - out[9] = (w3 >> 6) & mask; - out[10] = (w3 >> 28) & mask; - out[11] = ((w3 >> 50) | (w4 << 14)) & mask; - out[12] = (w4 >> 8) & mask; - out[13] = (w4 >> 30) & mask; - out[14] = ((w4 >> 52) | (w5 << 12)) & mask; - out[15] = (w5 >> 10) & mask; - out[16] = (w5 >> 32) & mask; - out[17] = ((w5 >> 54) | (w6 << 10)) & mask; - out[18] = (w6 >> 12) & mask; - out[19] = (w6 >> 34) & mask; - out[20] = ((w6 >> 56) | (w7 << 8)) & mask; - out[21] = (w7 >> 14) & mask; - out[22] = (w7 >> 36) & mask; - out[23] = ((w7 >> 58) | (w8 << 6)) & mask; - out[24] = (w8 >> 16) & mask; - out[25] = (w8 >> 38) & mask; - out[26] = ((w8 >> 60) | (w9 << 4)) & mask; - out[27] = (w9 >> 18) & mask; - out[28] = (w9 >> 40) & mask; - out[29] = ((w9 >> 62) | (w10 << 2)) & mask; - out[30] = (w10 >> 20) & mask; - out[31] = w10 >> 42; - - return in; -} - -inline const uint8_t* unpack23_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 8388607ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 4; - out[0] = (w0)&mask; - out[1] = (w0 >> 23) & mask; - out[2] = ((w0 >> 46) | (w1 << 18)) & mask; - out[3] = (w1 >> 5) & mask; - out[4] = (w1 >> 28) & mask; - out[5] = ((w1 >> 51) | (w2 << 13)) & mask; - out[6] = (w2 >> 10) & mask; - out[7] = (w2 >> 33) & mask; - out[8] = ((w2 >> 56) | (w3 << 8)) & mask; - out[9] = (w3 >> 15) & mask; - out[10] = (w3 >> 38) & mask; - out[11] = ((w3 >> 61) | (w4 << 3)) & mask; - out[12] = (w4 >> 20) & mask; - out[13] = ((w4 >> 43) | (w5 << 21)) & mask; - out[14] = (w5 >> 2) & mask; - out[15] = (w5 >> 25) & mask; - out[16] = ((w5 >> 48) | (w6 << 16)) & mask; - out[17] = (w6 >> 7) & mask; - out[18] = (w6 >> 30) & mask; - out[19] = ((w6 >> 53) | (w7 << 11)) & mask; - out[20] = (w7 >> 12) & mask; - out[21] = (w7 >> 35) & mask; - out[22] = ((w7 >> 58) | (w8 << 6)) & mask; - out[23] = (w8 >> 17) & mask; - out[24] = (w8 >> 40) & mask; - out[25] = ((w8 >> 63) | (w9 << 1)) & mask; - out[26] = (w9 >> 22) & mask; - out[27] = ((w9 >> 45) | (w10 << 19)) & mask; - out[28] = (w10 >> 4) & mask; - out[29] = (w10 >> 27) & mask; - out[30] = ((w10 >> 50) | (w11 << 14)) & mask; - out[31] = (w11 >> 9) & mask; - - return in; -} - -inline const uint8_t* unpack24_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 16777215ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - out[0] = (w0)&mask; - out[1] = (w0 >> 24) & mask; - out[2] = ((w0 >> 48) | (w1 << 16)) & mask; - out[3] = (w1 >> 8) & mask; - out[4] = (w1 >> 32) & mask; - out[5] = ((w1 >> 56) | (w2 << 8)) & mask; - out[6] = (w2 >> 16) & mask; - out[7] = w2 >> 40; - out[8] = (w3)&mask; - out[9] = (w3 >> 24) & mask; - out[10] = ((w3 >> 48) | (w4 << 16)) & mask; - out[11] = (w4 >> 8) & mask; - out[12] = (w4 >> 32) & mask; - out[13] = ((w4 >> 56) | (w5 << 8)) & mask; - out[14] = (w5 >> 16) & mask; - out[15] = w5 >> 40; - out[16] = (w6)&mask; - out[17] = (w6 >> 24) & mask; - out[18] = ((w6 >> 48) | (w7 << 16)) & mask; - out[19] = (w7 >> 8) & mask; - out[20] = (w7 >> 32) & mask; - out[21] = ((w7 >> 56) | (w8 << 8)) & mask; - out[22] = (w8 >> 16) & mask; - out[23] = w8 >> 40; - out[24] = (w9)&mask; - out[25] = (w9 >> 24) & mask; - out[26] = ((w9 >> 48) | (w10 << 16)) & mask; - out[27] = (w10 >> 8) & mask; - out[28] = (w10 >> 32) & mask; - out[29] = ((w10 >> 56) | (w11 << 8)) & mask; - out[30] = (w11 >> 16) & mask; - out[31] = w11 >> 40; - - return in; -} - -inline const uint8_t* unpack25_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 33554431ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 4; - out[0] = (w0)&mask; - out[1] = (w0 >> 25) & mask; - out[2] = ((w0 >> 50) | (w1 << 14)) & mask; - out[3] = (w1 >> 11) & mask; - out[4] = (w1 >> 36) & mask; - out[5] = ((w1 >> 61) | (w2 << 3)) & mask; - out[6] = (w2 >> 22) & mask; - out[7] = ((w2 >> 47) | (w3 << 17)) & mask; - out[8] = (w3 >> 8) & mask; - out[9] = (w3 >> 33) & mask; - out[10] = ((w3 >> 58) | (w4 << 6)) & mask; - out[11] = (w4 >> 19) & mask; - out[12] = ((w4 >> 44) | (w5 << 20)) & mask; - out[13] = (w5 >> 5) & mask; - out[14] = (w5 >> 30) & mask; - out[15] = ((w5 >> 55) | (w6 << 9)) & mask; - out[16] = (w6 >> 16) & mask; - out[17] = ((w6 >> 41) | (w7 << 23)) & mask; - out[18] = (w7 >> 2) & mask; - out[19] = (w7 >> 27) & mask; - out[20] = ((w7 >> 52) | (w8 << 12)) & mask; - out[21] = (w8 >> 13) & mask; - out[22] = (w8 >> 38) & mask; - out[23] = ((w8 >> 63) | (w9 << 1)) & mask; - out[24] = (w9 >> 24) & mask; - out[25] = ((w9 >> 49) | (w10 << 15)) & mask; - out[26] = (w10 >> 10) & mask; - out[27] = (w10 >> 35) & mask; - out[28] = ((w10 >> 60) | (w11 << 4)) & mask; - out[29] = (w11 >> 21) & mask; - out[30] = ((w11 >> 46) | (w12 << 18)) & mask; - out[31] = (w12 >> 7) & mask; - - return in; -} - -inline const uint8_t* unpack26_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 67108863ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - out[0] = (w0)&mask; - out[1] = (w0 >> 26) & mask; - out[2] = ((w0 >> 52) | (w1 << 12)) & mask; - out[3] = (w1 >> 14) & mask; - out[4] = ((w1 >> 40) | (w2 << 24)) & mask; - out[5] = (w2 >> 2) & mask; - out[6] = (w2 >> 28) & mask; - out[7] = ((w2 >> 54) | (w3 << 10)) & mask; - out[8] = (w3 >> 16) & mask; - out[9] = ((w3 >> 42) | (w4 << 22)) & mask; - out[10] = (w4 >> 4) & mask; - out[11] = (w4 >> 30) & mask; - out[12] = ((w4 >> 56) | (w5 << 8)) & mask; - out[13] = (w5 >> 18) & mask; - out[14] = ((w5 >> 44) | (w6 << 20)) & mask; - out[15] = (w6 >> 6) & mask; - out[16] = (w6 >> 32) & mask; - out[17] = ((w6 >> 58) | (w7 << 6)) & mask; - out[18] = (w7 >> 20) & mask; - out[19] = ((w7 >> 46) | (w8 << 18)) & mask; - out[20] = (w8 >> 8) & mask; - out[21] = (w8 >> 34) & mask; - out[22] = ((w8 >> 60) | (w9 << 4)) & mask; - out[23] = (w9 >> 22) & mask; - out[24] = ((w9 >> 48) | (w10 << 16)) & mask; - out[25] = (w10 >> 10) & mask; - out[26] = (w10 >> 36) & mask; - out[27] = ((w10 >> 62) | (w11 << 2)) & mask; - out[28] = (w11 >> 24) & mask; - out[29] = ((w11 >> 50) | (w12 << 14)) & mask; - out[30] = (w12 >> 12) & mask; - out[31] = w12 >> 38; - - return in; -} - -inline const uint8_t* unpack27_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 134217727ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 4; - out[0] = (w0)&mask; - out[1] = (w0 >> 27) & mask; - out[2] = ((w0 >> 54) | (w1 << 10)) & mask; - out[3] = (w1 >> 17) & mask; - out[4] = ((w1 >> 44) | (w2 << 20)) & mask; - out[5] = (w2 >> 7) & mask; - out[6] = (w2 >> 34) & mask; - out[7] = ((w2 >> 61) | (w3 << 3)) & mask; - out[8] = (w3 >> 24) & mask; - out[9] = ((w3 >> 51) | (w4 << 13)) & mask; - out[10] = (w4 >> 14) & mask; - out[11] = ((w4 >> 41) | (w5 << 23)) & mask; - out[12] = (w5 >> 4) & mask; - out[13] = (w5 >> 31) & mask; - out[14] = ((w5 >> 58) | (w6 << 6)) & mask; - out[15] = (w6 >> 21) & mask; - out[16] = ((w6 >> 48) | (w7 << 16)) & mask; - out[17] = (w7 >> 11) & mask; - out[18] = ((w7 >> 38) | (w8 << 26)) & mask; - out[19] = (w8 >> 1) & mask; - out[20] = (w8 >> 28) & mask; - out[21] = ((w8 >> 55) | (w9 << 9)) & mask; - out[22] = (w9 >> 18) & mask; - out[23] = ((w9 >> 45) | (w10 << 19)) & mask; - out[24] = (w10 >> 8) & mask; - out[25] = (w10 >> 35) & mask; - out[26] = ((w10 >> 62) | (w11 << 2)) & mask; - out[27] = (w11 >> 25) & mask; - out[28] = ((w11 >> 52) | (w12 << 12)) & mask; - out[29] = (w12 >> 15) & mask; - out[30] = ((w12 >> 42) | (w13 << 22)) & mask; - out[31] = (w13 >> 5) & mask; - - return in; -} - -inline const uint8_t* unpack28_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 268435455ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - out[0] = (w0)&mask; - out[1] = (w0 >> 28) & mask; - out[2] = ((w0 >> 56) | (w1 << 8)) & mask; - out[3] = (w1 >> 20) & mask; - out[4] = ((w1 >> 48) | (w2 << 16)) & mask; - out[5] = (w2 >> 12) & mask; - out[6] = ((w2 >> 40) | (w3 << 24)) & mask; - out[7] = (w3 >> 4) & mask; - out[8] = (w3 >> 32) & mask; - out[9] = ((w3 >> 60) | (w4 << 4)) & mask; - out[10] = (w4 >> 24) & mask; - out[11] = ((w4 >> 52) | (w5 << 12)) & mask; - out[12] = (w5 >> 16) & mask; - out[13] = ((w5 >> 44) | (w6 << 20)) & mask; - out[14] = (w6 >> 8) & mask; - out[15] = w6 >> 36; - out[16] = (w7)&mask; - out[17] = (w7 >> 28) & mask; - out[18] = ((w7 >> 56) | (w8 << 8)) & mask; - out[19] = (w8 >> 20) & mask; - out[20] = ((w8 >> 48) | (w9 << 16)) & mask; - out[21] = (w9 >> 12) & mask; - out[22] = ((w9 >> 40) | (w10 << 24)) & mask; - out[23] = (w10 >> 4) & mask; - out[24] = (w10 >> 32) & mask; - out[25] = ((w10 >> 60) | (w11 << 4)) & mask; - out[26] = (w11 >> 24) & mask; - out[27] = ((w11 >> 52) | (w12 << 12)) & mask; - out[28] = (w12 >> 16) & mask; - out[29] = ((w12 >> 44) | (w13 << 20)) & mask; - out[30] = (w13 >> 8) & mask; - out[31] = w13 >> 36; - - return in; -} - -inline const uint8_t* unpack29_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 536870911ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 4; - out[0] = (w0)&mask; - out[1] = (w0 >> 29) & mask; - out[2] = ((w0 >> 58) | (w1 << 6)) & mask; - out[3] = (w1 >> 23) & mask; - out[4] = ((w1 >> 52) | (w2 << 12)) & mask; - out[5] = (w2 >> 17) & mask; - out[6] = ((w2 >> 46) | (w3 << 18)) & mask; - out[7] = (w3 >> 11) & mask; - out[8] = ((w3 >> 40) | (w4 << 24)) & mask; - out[9] = (w4 >> 5) & mask; - out[10] = (w4 >> 34) & mask; - out[11] = ((w4 >> 63) | (w5 << 1)) & mask; - out[12] = (w5 >> 28) & mask; - out[13] = ((w5 >> 57) | (w6 << 7)) & mask; - out[14] = (w6 >> 22) & mask; - out[15] = ((w6 >> 51) | (w7 << 13)) & mask; - out[16] = (w7 >> 16) & mask; - out[17] = ((w7 >> 45) | (w8 << 19)) & mask; - out[18] = (w8 >> 10) & mask; - out[19] = ((w8 >> 39) | (w9 << 25)) & mask; - out[20] = (w9 >> 4) & mask; - out[21] = (w9 >> 33) & mask; - out[22] = ((w9 >> 62) | (w10 << 2)) & mask; - out[23] = (w10 >> 27) & mask; - out[24] = ((w10 >> 56) | (w11 << 8)) & mask; - out[25] = (w11 >> 21) & mask; - out[26] = ((w11 >> 50) | (w12 << 14)) & mask; - out[27] = (w12 >> 15) & mask; - out[28] = ((w12 >> 44) | (w13 << 20)) & mask; - out[29] = (w13 >> 9) & mask; - out[30] = ((w13 >> 38) | (w14 << 26)) & mask; - out[31] = (w14 >> 3) & mask; - - return in; -} - -inline const uint8_t* unpack30_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 1073741823ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - out[0] = (w0)&mask; - out[1] = (w0 >> 30) & mask; - out[2] = ((w0 >> 60) | (w1 << 4)) & mask; - out[3] = (w1 >> 26) & mask; - out[4] = ((w1 >> 56) | (w2 << 8)) & mask; - out[5] = (w2 >> 22) & mask; - out[6] = ((w2 >> 52) | (w3 << 12)) & mask; - out[7] = (w3 >> 18) & mask; - out[8] = ((w3 >> 48) | (w4 << 16)) & mask; - out[9] = (w4 >> 14) & mask; - out[10] = ((w4 >> 44) | (w5 << 20)) & mask; - out[11] = (w5 >> 10) & mask; - out[12] = ((w5 >> 40) | (w6 << 24)) & mask; - out[13] = (w6 >> 6) & mask; - out[14] = ((w6 >> 36) | (w7 << 28)) & mask; - out[15] = (w7 >> 2) & mask; - out[16] = (w7 >> 32) & mask; - out[17] = ((w7 >> 62) | (w8 << 2)) & mask; - out[18] = (w8 >> 28) & mask; - out[19] = ((w8 >> 58) | (w9 << 6)) & mask; - out[20] = (w9 >> 24) & mask; - out[21] = ((w9 >> 54) | (w10 << 10)) & mask; - out[22] = (w10 >> 20) & mask; - out[23] = ((w10 >> 50) | (w11 << 14)) & mask; - out[24] = (w11 >> 16) & mask; - out[25] = ((w11 >> 46) | (w12 << 18)) & mask; - out[26] = (w12 >> 12) & mask; - out[27] = ((w12 >> 42) | (w13 << 22)) & mask; - out[28] = (w13 >> 8) & mask; - out[29] = ((w13 >> 38) | (w14 << 26)) & mask; - out[30] = (w14 >> 4) & mask; - out[31] = w14 >> 34; - - return in; -} - -inline const uint8_t* unpack31_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 2147483647ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 4; - out[0] = (w0)&mask; - out[1] = (w0 >> 31) & mask; - out[2] = ((w0 >> 62) | (w1 << 2)) & mask; - out[3] = (w1 >> 29) & mask; - out[4] = ((w1 >> 60) | (w2 << 4)) & mask; - out[5] = (w2 >> 27) & mask; - out[6] = ((w2 >> 58) | (w3 << 6)) & mask; - out[7] = (w3 >> 25) & mask; - out[8] = ((w3 >> 56) | (w4 << 8)) & mask; - out[9] = (w4 >> 23) & mask; - out[10] = ((w4 >> 54) | (w5 << 10)) & mask; - out[11] = (w5 >> 21) & mask; - out[12] = ((w5 >> 52) | (w6 << 12)) & mask; - out[13] = (w6 >> 19) & mask; - out[14] = ((w6 >> 50) | (w7 << 14)) & mask; - out[15] = (w7 >> 17) & mask; - out[16] = ((w7 >> 48) | (w8 << 16)) & mask; - out[17] = (w8 >> 15) & mask; - out[18] = ((w8 >> 46) | (w9 << 18)) & mask; - out[19] = (w9 >> 13) & mask; - out[20] = ((w9 >> 44) | (w10 << 20)) & mask; - out[21] = (w10 >> 11) & mask; - out[22] = ((w10 >> 42) | (w11 << 22)) & mask; - out[23] = (w11 >> 9) & mask; - out[24] = ((w11 >> 40) | (w12 << 24)) & mask; - out[25] = (w12 >> 7) & mask; - out[26] = ((w12 >> 38) | (w13 << 26)) & mask; - out[27] = (w13 >> 5) & mask; - out[28] = ((w13 >> 36) | (w14 << 28)) & mask; - out[29] = (w14 >> 3) & mask; - out[30] = ((w14 >> 34) | (w15 << 30)) & mask; - out[31] = (w15 >> 1) & mask; - - return in; -} - -inline const uint8_t* unpack32_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 4294967295ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - out[0] = (w0)&mask; - out[1] = w0 >> 32; - out[2] = (w1)&mask; - out[3] = w1 >> 32; - out[4] = (w2)&mask; - out[5] = w2 >> 32; - out[6] = (w3)&mask; - out[7] = w3 >> 32; - out[8] = (w4)&mask; - out[9] = w4 >> 32; - out[10] = (w5)&mask; - out[11] = w5 >> 32; - out[12] = (w6)&mask; - out[13] = w6 >> 32; - out[14] = (w7)&mask; - out[15] = w7 >> 32; - out[16] = (w8)&mask; - out[17] = w8 >> 32; - out[18] = (w9)&mask; - out[19] = w9 >> 32; - out[20] = (w10)&mask; - out[21] = w10 >> 32; - out[22] = (w11)&mask; - out[23] = w11 >> 32; - out[24] = (w12)&mask; - out[25] = w12 >> 32; - out[26] = (w13)&mask; - out[27] = w13 >> 32; - out[28] = (w14)&mask; - out[29] = w14 >> 32; - out[30] = (w15)&mask; - out[31] = w15 >> 32; - - return in; -} - -inline const uint8_t* unpack33_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 8589934591ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 4; - out[0] = (w0)&mask; - out[1] = ((w0 >> 33) | (w1 << 31)) & mask; - out[2] = (w1 >> 2) & mask; - out[3] = ((w1 >> 35) | (w2 << 29)) & mask; - out[4] = (w2 >> 4) & mask; - out[5] = ((w2 >> 37) | (w3 << 27)) & mask; - out[6] = (w3 >> 6) & mask; - out[7] = ((w3 >> 39) | (w4 << 25)) & mask; - out[8] = (w4 >> 8) & mask; - out[9] = ((w4 >> 41) | (w5 << 23)) & mask; - out[10] = (w5 >> 10) & mask; - out[11] = ((w5 >> 43) | (w6 << 21)) & mask; - out[12] = (w6 >> 12) & mask; - out[13] = ((w6 >> 45) | (w7 << 19)) & mask; - out[14] = (w7 >> 14) & mask; - out[15] = ((w7 >> 47) | (w8 << 17)) & mask; - out[16] = (w8 >> 16) & mask; - out[17] = ((w8 >> 49) | (w9 << 15)) & mask; - out[18] = (w9 >> 18) & mask; - out[19] = ((w9 >> 51) | (w10 << 13)) & mask; - out[20] = (w10 >> 20) & mask; - out[21] = ((w10 >> 53) | (w11 << 11)) & mask; - out[22] = (w11 >> 22) & mask; - out[23] = ((w11 >> 55) | (w12 << 9)) & mask; - out[24] = (w12 >> 24) & mask; - out[25] = ((w12 >> 57) | (w13 << 7)) & mask; - out[26] = (w13 >> 26) & mask; - out[27] = ((w13 >> 59) | (w14 << 5)) & mask; - out[28] = (w14 >> 28) & mask; - out[29] = ((w14 >> 61) | (w15 << 3)) & mask; - out[30] = (w15 >> 30) & mask; - out[31] = ((w15 >> 63) | (w16 << 1)) & mask; - - return in; -} - -inline const uint8_t* unpack34_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 17179869183ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - out[0] = (w0)&mask; - out[1] = ((w0 >> 34) | (w1 << 30)) & mask; - out[2] = (w1 >> 4) & mask; - out[3] = ((w1 >> 38) | (w2 << 26)) & mask; - out[4] = (w2 >> 8) & mask; - out[5] = ((w2 >> 42) | (w3 << 22)) & mask; - out[6] = (w3 >> 12) & mask; - out[7] = ((w3 >> 46) | (w4 << 18)) & mask; - out[8] = (w4 >> 16) & mask; - out[9] = ((w4 >> 50) | (w5 << 14)) & mask; - out[10] = (w5 >> 20) & mask; - out[11] = ((w5 >> 54) | (w6 << 10)) & mask; - out[12] = (w6 >> 24) & mask; - out[13] = ((w6 >> 58) | (w7 << 6)) & mask; - out[14] = (w7 >> 28) & mask; - out[15] = ((w7 >> 62) | (w8 << 2)) & mask; - out[16] = ((w8 >> 32) | (w9 << 32)) & mask; - out[17] = (w9 >> 2) & mask; - out[18] = ((w9 >> 36) | (w10 << 28)) & mask; - out[19] = (w10 >> 6) & mask; - out[20] = ((w10 >> 40) | (w11 << 24)) & mask; - out[21] = (w11 >> 10) & mask; - out[22] = ((w11 >> 44) | (w12 << 20)) & mask; - out[23] = (w12 >> 14) & mask; - out[24] = ((w12 >> 48) | (w13 << 16)) & mask; - out[25] = (w13 >> 18) & mask; - out[26] = ((w13 >> 52) | (w14 << 12)) & mask; - out[27] = (w14 >> 22) & mask; - out[28] = ((w14 >> 56) | (w15 << 8)) & mask; - out[29] = (w15 >> 26) & mask; - out[30] = ((w15 >> 60) | (w16 << 4)) & mask; - out[31] = w16 >> 30; - - return in; -} - -inline const uint8_t* unpack35_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 34359738367ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 4; - out[0] = (w0)&mask; - out[1] = ((w0 >> 35) | (w1 << 29)) & mask; - out[2] = (w1 >> 6) & mask; - out[3] = ((w1 >> 41) | (w2 << 23)) & mask; - out[4] = (w2 >> 12) & mask; - out[5] = ((w2 >> 47) | (w3 << 17)) & mask; - out[6] = (w3 >> 18) & mask; - out[7] = ((w3 >> 53) | (w4 << 11)) & mask; - out[8] = (w4 >> 24) & mask; - out[9] = ((w4 >> 59) | (w5 << 5)) & mask; - out[10] = ((w5 >> 30) | (w6 << 34)) & mask; - out[11] = (w6 >> 1) & mask; - out[12] = ((w6 >> 36) | (w7 << 28)) & mask; - out[13] = (w7 >> 7) & mask; - out[14] = ((w7 >> 42) | (w8 << 22)) & mask; - out[15] = (w8 >> 13) & mask; - out[16] = ((w8 >> 48) | (w9 << 16)) & mask; - out[17] = (w9 >> 19) & mask; - out[18] = ((w9 >> 54) | (w10 << 10)) & mask; - out[19] = (w10 >> 25) & mask; - out[20] = ((w10 >> 60) | (w11 << 4)) & mask; - out[21] = ((w11 >> 31) | (w12 << 33)) & mask; - out[22] = (w12 >> 2) & mask; - out[23] = ((w12 >> 37) | (w13 << 27)) & mask; - out[24] = (w13 >> 8) & mask; - out[25] = ((w13 >> 43) | (w14 << 21)) & mask; - out[26] = (w14 >> 14) & mask; - out[27] = ((w14 >> 49) | (w15 << 15)) & mask; - out[28] = (w15 >> 20) & mask; - out[29] = ((w15 >> 55) | (w16 << 9)) & mask; - out[30] = (w16 >> 26) & mask; - out[31] = ((w16 >> 61) | (w17 << 3)) & mask; - - return in; -} - -inline const uint8_t* unpack36_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 68719476735ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - out[0] = (w0)&mask; - out[1] = ((w0 >> 36) | (w1 << 28)) & mask; - out[2] = (w1 >> 8) & mask; - out[3] = ((w1 >> 44) | (w2 << 20)) & mask; - out[4] = (w2 >> 16) & mask; - out[5] = ((w2 >> 52) | (w3 << 12)) & mask; - out[6] = (w3 >> 24) & mask; - out[7] = ((w3 >> 60) | (w4 << 4)) & mask; - out[8] = ((w4 >> 32) | (w5 << 32)) & mask; - out[9] = (w5 >> 4) & mask; - out[10] = ((w5 >> 40) | (w6 << 24)) & mask; - out[11] = (w6 >> 12) & mask; - out[12] = ((w6 >> 48) | (w7 << 16)) & mask; - out[13] = (w7 >> 20) & mask; - out[14] = ((w7 >> 56) | (w8 << 8)) & mask; - out[15] = w8 >> 28; - out[16] = (w9)&mask; - out[17] = ((w9 >> 36) | (w10 << 28)) & mask; - out[18] = (w10 >> 8) & mask; - out[19] = ((w10 >> 44) | (w11 << 20)) & mask; - out[20] = (w11 >> 16) & mask; - out[21] = ((w11 >> 52) | (w12 << 12)) & mask; - out[22] = (w12 >> 24) & mask; - out[23] = ((w12 >> 60) | (w13 << 4)) & mask; - out[24] = ((w13 >> 32) | (w14 << 32)) & mask; - out[25] = (w14 >> 4) & mask; - out[26] = ((w14 >> 40) | (w15 << 24)) & mask; - out[27] = (w15 >> 12) & mask; - out[28] = ((w15 >> 48) | (w16 << 16)) & mask; - out[29] = (w16 >> 20) & mask; - out[30] = ((w16 >> 56) | (w17 << 8)) & mask; - out[31] = w17 >> 28; - - return in; -} - -inline const uint8_t* unpack37_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 137438953471ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 4; - out[0] = (w0)&mask; - out[1] = ((w0 >> 37) | (w1 << 27)) & mask; - out[2] = (w1 >> 10) & mask; - out[3] = ((w1 >> 47) | (w2 << 17)) & mask; - out[4] = (w2 >> 20) & mask; - out[5] = ((w2 >> 57) | (w3 << 7)) & mask; - out[6] = ((w3 >> 30) | (w4 << 34)) & mask; - out[7] = (w4 >> 3) & mask; - out[8] = ((w4 >> 40) | (w5 << 24)) & mask; - out[9] = (w5 >> 13) & mask; - out[10] = ((w5 >> 50) | (w6 << 14)) & mask; - out[11] = (w6 >> 23) & mask; - out[12] = ((w6 >> 60) | (w7 << 4)) & mask; - out[13] = ((w7 >> 33) | (w8 << 31)) & mask; - out[14] = (w8 >> 6) & mask; - out[15] = ((w8 >> 43) | (w9 << 21)) & mask; - out[16] = (w9 >> 16) & mask; - out[17] = ((w9 >> 53) | (w10 << 11)) & mask; - out[18] = (w10 >> 26) & mask; - out[19] = ((w10 >> 63) | (w11 << 1)) & mask; - out[20] = ((w11 >> 36) | (w12 << 28)) & mask; - out[21] = (w12 >> 9) & mask; - out[22] = ((w12 >> 46) | (w13 << 18)) & mask; - out[23] = (w13 >> 19) & mask; - out[24] = ((w13 >> 56) | (w14 << 8)) & mask; - out[25] = ((w14 >> 29) | (w15 << 35)) & mask; - out[26] = (w15 >> 2) & mask; - out[27] = ((w15 >> 39) | (w16 << 25)) & mask; - out[28] = (w16 >> 12) & mask; - out[29] = ((w16 >> 49) | (w17 << 15)) & mask; - out[30] = (w17 >> 22) & mask; - out[31] = ((w17 >> 59) | (w18 << 5)) & mask; - - return in; -} - -inline const uint8_t* unpack38_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 274877906943ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - out[0] = (w0)&mask; - out[1] = ((w0 >> 38) | (w1 << 26)) & mask; - out[2] = (w1 >> 12) & mask; - out[3] = ((w1 >> 50) | (w2 << 14)) & mask; - out[4] = (w2 >> 24) & mask; - out[5] = ((w2 >> 62) | (w3 << 2)) & mask; - out[6] = ((w3 >> 36) | (w4 << 28)) & mask; - out[7] = (w4 >> 10) & mask; - out[8] = ((w4 >> 48) | (w5 << 16)) & mask; - out[9] = (w5 >> 22) & mask; - out[10] = ((w5 >> 60) | (w6 << 4)) & mask; - out[11] = ((w6 >> 34) | (w7 << 30)) & mask; - out[12] = (w7 >> 8) & mask; - out[13] = ((w7 >> 46) | (w8 << 18)) & mask; - out[14] = (w8 >> 20) & mask; - out[15] = ((w8 >> 58) | (w9 << 6)) & mask; - out[16] = ((w9 >> 32) | (w10 << 32)) & mask; - out[17] = (w10 >> 6) & mask; - out[18] = ((w10 >> 44) | (w11 << 20)) & mask; - out[19] = (w11 >> 18) & mask; - out[20] = ((w11 >> 56) | (w12 << 8)) & mask; - out[21] = ((w12 >> 30) | (w13 << 34)) & mask; - out[22] = (w13 >> 4) & mask; - out[23] = ((w13 >> 42) | (w14 << 22)) & mask; - out[24] = (w14 >> 16) & mask; - out[25] = ((w14 >> 54) | (w15 << 10)) & mask; - out[26] = ((w15 >> 28) | (w16 << 36)) & mask; - out[27] = (w16 >> 2) & mask; - out[28] = ((w16 >> 40) | (w17 << 24)) & mask; - out[29] = (w17 >> 14) & mask; - out[30] = ((w17 >> 52) | (w18 << 12)) & mask; - out[31] = w18 >> 26; - - return in; -} - -inline const uint8_t* unpack39_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 549755813887ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 4; - out[0] = (w0)&mask; - out[1] = ((w0 >> 39) | (w1 << 25)) & mask; - out[2] = (w1 >> 14) & mask; - out[3] = ((w1 >> 53) | (w2 << 11)) & mask; - out[4] = ((w2 >> 28) | (w3 << 36)) & mask; - out[5] = (w3 >> 3) & mask; - out[6] = ((w3 >> 42) | (w4 << 22)) & mask; - out[7] = (w4 >> 17) & mask; - out[8] = ((w4 >> 56) | (w5 << 8)) & mask; - out[9] = ((w5 >> 31) | (w6 << 33)) & mask; - out[10] = (w6 >> 6) & mask; - out[11] = ((w6 >> 45) | (w7 << 19)) & mask; - out[12] = (w7 >> 20) & mask; - out[13] = ((w7 >> 59) | (w8 << 5)) & mask; - out[14] = ((w8 >> 34) | (w9 << 30)) & mask; - out[15] = (w9 >> 9) & mask; - out[16] = ((w9 >> 48) | (w10 << 16)) & mask; - out[17] = (w10 >> 23) & mask; - out[18] = ((w10 >> 62) | (w11 << 2)) & mask; - out[19] = ((w11 >> 37) | (w12 << 27)) & mask; - out[20] = (w12 >> 12) & mask; - out[21] = ((w12 >> 51) | (w13 << 13)) & mask; - out[22] = ((w13 >> 26) | (w14 << 38)) & mask; - out[23] = (w14 >> 1) & mask; - out[24] = ((w14 >> 40) | (w15 << 24)) & mask; - out[25] = (w15 >> 15) & mask; - out[26] = ((w15 >> 54) | (w16 << 10)) & mask; - out[27] = ((w16 >> 29) | (w17 << 35)) & mask; - out[28] = (w17 >> 4) & mask; - out[29] = ((w17 >> 43) | (w18 << 21)) & mask; - out[30] = (w18 >> 18) & mask; - out[31] = ((w18 >> 57) | (w19 << 7)) & mask; - - return in; -} - -inline const uint8_t* unpack40_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 1099511627775ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - out[0] = (w0)&mask; - out[1] = ((w0 >> 40) | (w1 << 24)) & mask; - out[2] = (w1 >> 16) & mask; - out[3] = ((w1 >> 56) | (w2 << 8)) & mask; - out[4] = ((w2 >> 32) | (w3 << 32)) & mask; - out[5] = (w3 >> 8) & mask; - out[6] = ((w3 >> 48) | (w4 << 16)) & mask; - out[7] = w4 >> 24; - out[8] = (w5)&mask; - out[9] = ((w5 >> 40) | (w6 << 24)) & mask; - out[10] = (w6 >> 16) & mask; - out[11] = ((w6 >> 56) | (w7 << 8)) & mask; - out[12] = ((w7 >> 32) | (w8 << 32)) & mask; - out[13] = (w8 >> 8) & mask; - out[14] = ((w8 >> 48) | (w9 << 16)) & mask; - out[15] = w9 >> 24; - out[16] = (w10)&mask; - out[17] = ((w10 >> 40) | (w11 << 24)) & mask; - out[18] = (w11 >> 16) & mask; - out[19] = ((w11 >> 56) | (w12 << 8)) & mask; - out[20] = ((w12 >> 32) | (w13 << 32)) & mask; - out[21] = (w13 >> 8) & mask; - out[22] = ((w13 >> 48) | (w14 << 16)) & mask; - out[23] = w14 >> 24; - out[24] = (w15)&mask; - out[25] = ((w15 >> 40) | (w16 << 24)) & mask; - out[26] = (w16 >> 16) & mask; - out[27] = ((w16 >> 56) | (w17 << 8)) & mask; - out[28] = ((w17 >> 32) | (w18 << 32)) & mask; - out[29] = (w18 >> 8) & mask; - out[30] = ((w18 >> 48) | (w19 << 16)) & mask; - out[31] = w19 >> 24; - - return in; -} - -inline const uint8_t* unpack41_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 2199023255551ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 4; - out[0] = (w0)&mask; - out[1] = ((w0 >> 41) | (w1 << 23)) & mask; - out[2] = (w1 >> 18) & mask; - out[3] = ((w1 >> 59) | (w2 << 5)) & mask; - out[4] = ((w2 >> 36) | (w3 << 28)) & mask; - out[5] = (w3 >> 13) & mask; - out[6] = ((w3 >> 54) | (w4 << 10)) & mask; - out[7] = ((w4 >> 31) | (w5 << 33)) & mask; - out[8] = (w5 >> 8) & mask; - out[9] = ((w5 >> 49) | (w6 << 15)) & mask; - out[10] = ((w6 >> 26) | (w7 << 38)) & mask; - out[11] = (w7 >> 3) & mask; - out[12] = ((w7 >> 44) | (w8 << 20)) & mask; - out[13] = (w8 >> 21) & mask; - out[14] = ((w8 >> 62) | (w9 << 2)) & mask; - out[15] = ((w9 >> 39) | (w10 << 25)) & mask; - out[16] = (w10 >> 16) & mask; - out[17] = ((w10 >> 57) | (w11 << 7)) & mask; - out[18] = ((w11 >> 34) | (w12 << 30)) & mask; - out[19] = (w12 >> 11) & mask; - out[20] = ((w12 >> 52) | (w13 << 12)) & mask; - out[21] = ((w13 >> 29) | (w14 << 35)) & mask; - out[22] = (w14 >> 6) & mask; - out[23] = ((w14 >> 47) | (w15 << 17)) & mask; - out[24] = ((w15 >> 24) | (w16 << 40)) & mask; - out[25] = (w16 >> 1) & mask; - out[26] = ((w16 >> 42) | (w17 << 22)) & mask; - out[27] = (w17 >> 19) & mask; - out[28] = ((w17 >> 60) | (w18 << 4)) & mask; - out[29] = ((w18 >> 37) | (w19 << 27)) & mask; - out[30] = (w19 >> 14) & mask; - out[31] = ((w19 >> 55) | (w20 << 9)) & mask; - - return in; -} - -inline const uint8_t* unpack42_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 4398046511103ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - out[0] = (w0)&mask; - out[1] = ((w0 >> 42) | (w1 << 22)) & mask; - out[2] = (w1 >> 20) & mask; - out[3] = ((w1 >> 62) | (w2 << 2)) & mask; - out[4] = ((w2 >> 40) | (w3 << 24)) & mask; - out[5] = (w3 >> 18) & mask; - out[6] = ((w3 >> 60) | (w4 << 4)) & mask; - out[7] = ((w4 >> 38) | (w5 << 26)) & mask; - out[8] = (w5 >> 16) & mask; - out[9] = ((w5 >> 58) | (w6 << 6)) & mask; - out[10] = ((w6 >> 36) | (w7 << 28)) & mask; - out[11] = (w7 >> 14) & mask; - out[12] = ((w7 >> 56) | (w8 << 8)) & mask; - out[13] = ((w8 >> 34) | (w9 << 30)) & mask; - out[14] = (w9 >> 12) & mask; - out[15] = ((w9 >> 54) | (w10 << 10)) & mask; - out[16] = ((w10 >> 32) | (w11 << 32)) & mask; - out[17] = (w11 >> 10) & mask; - out[18] = ((w11 >> 52) | (w12 << 12)) & mask; - out[19] = ((w12 >> 30) | (w13 << 34)) & mask; - out[20] = (w13 >> 8) & mask; - out[21] = ((w13 >> 50) | (w14 << 14)) & mask; - out[22] = ((w14 >> 28) | (w15 << 36)) & mask; - out[23] = (w15 >> 6) & mask; - out[24] = ((w15 >> 48) | (w16 << 16)) & mask; - out[25] = ((w16 >> 26) | (w17 << 38)) & mask; - out[26] = (w17 >> 4) & mask; - out[27] = ((w17 >> 46) | (w18 << 18)) & mask; - out[28] = ((w18 >> 24) | (w19 << 40)) & mask; - out[29] = (w19 >> 2) & mask; - out[30] = ((w19 >> 44) | (w20 << 20)) & mask; - out[31] = w20 >> 22; - - return in; -} - -inline const uint8_t* unpack43_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 8796093022207ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - uint64_t w21 = util::SafeLoadAs(in); - w21 = arrow::bit_util::FromLittleEndian(w21); - in += 4; - out[0] = (w0)&mask; - out[1] = ((w0 >> 43) | (w1 << 21)) & mask; - out[2] = ((w1 >> 22) | (w2 << 42)) & mask; - out[3] = (w2 >> 1) & mask; - out[4] = ((w2 >> 44) | (w3 << 20)) & mask; - out[5] = ((w3 >> 23) | (w4 << 41)) & mask; - out[6] = (w4 >> 2) & mask; - out[7] = ((w4 >> 45) | (w5 << 19)) & mask; - out[8] = ((w5 >> 24) | (w6 << 40)) & mask; - out[9] = (w6 >> 3) & mask; - out[10] = ((w6 >> 46) | (w7 << 18)) & mask; - out[11] = ((w7 >> 25) | (w8 << 39)) & mask; - out[12] = (w8 >> 4) & mask; - out[13] = ((w8 >> 47) | (w9 << 17)) & mask; - out[14] = ((w9 >> 26) | (w10 << 38)) & mask; - out[15] = (w10 >> 5) & mask; - out[16] = ((w10 >> 48) | (w11 << 16)) & mask; - out[17] = ((w11 >> 27) | (w12 << 37)) & mask; - out[18] = (w12 >> 6) & mask; - out[19] = ((w12 >> 49) | (w13 << 15)) & mask; - out[20] = ((w13 >> 28) | (w14 << 36)) & mask; - out[21] = (w14 >> 7) & mask; - out[22] = ((w14 >> 50) | (w15 << 14)) & mask; - out[23] = ((w15 >> 29) | (w16 << 35)) & mask; - out[24] = (w16 >> 8) & mask; - out[25] = ((w16 >> 51) | (w17 << 13)) & mask; - out[26] = ((w17 >> 30) | (w18 << 34)) & mask; - out[27] = (w18 >> 9) & mask; - out[28] = ((w18 >> 52) | (w19 << 12)) & mask; - out[29] = ((w19 >> 31) | (w20 << 33)) & mask; - out[30] = (w20 >> 10) & mask; - out[31] = ((w20 >> 53) | (w21 << 11)) & mask; - - return in; -} - -inline const uint8_t* unpack44_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 17592186044415ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - uint64_t w21 = util::SafeLoadAs(in); - w21 = arrow::bit_util::FromLittleEndian(w21); - in += 8; - out[0] = (w0)&mask; - out[1] = ((w0 >> 44) | (w1 << 20)) & mask; - out[2] = ((w1 >> 24) | (w2 << 40)) & mask; - out[3] = (w2 >> 4) & mask; - out[4] = ((w2 >> 48) | (w3 << 16)) & mask; - out[5] = ((w3 >> 28) | (w4 << 36)) & mask; - out[6] = (w4 >> 8) & mask; - out[7] = ((w4 >> 52) | (w5 << 12)) & mask; - out[8] = ((w5 >> 32) | (w6 << 32)) & mask; - out[9] = (w6 >> 12) & mask; - out[10] = ((w6 >> 56) | (w7 << 8)) & mask; - out[11] = ((w7 >> 36) | (w8 << 28)) & mask; - out[12] = (w8 >> 16) & mask; - out[13] = ((w8 >> 60) | (w9 << 4)) & mask; - out[14] = ((w9 >> 40) | (w10 << 24)) & mask; - out[15] = w10 >> 20; - out[16] = (w11)&mask; - out[17] = ((w11 >> 44) | (w12 << 20)) & mask; - out[18] = ((w12 >> 24) | (w13 << 40)) & mask; - out[19] = (w13 >> 4) & mask; - out[20] = ((w13 >> 48) | (w14 << 16)) & mask; - out[21] = ((w14 >> 28) | (w15 << 36)) & mask; - out[22] = (w15 >> 8) & mask; - out[23] = ((w15 >> 52) | (w16 << 12)) & mask; - out[24] = ((w16 >> 32) | (w17 << 32)) & mask; - out[25] = (w17 >> 12) & mask; - out[26] = ((w17 >> 56) | (w18 << 8)) & mask; - out[27] = ((w18 >> 36) | (w19 << 28)) & mask; - out[28] = (w19 >> 16) & mask; - out[29] = ((w19 >> 60) | (w20 << 4)) & mask; - out[30] = ((w20 >> 40) | (w21 << 24)) & mask; - out[31] = w21 >> 20; - - return in; -} - -inline const uint8_t* unpack45_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 35184372088831ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - uint64_t w21 = util::SafeLoadAs(in); - w21 = arrow::bit_util::FromLittleEndian(w21); - in += 8; - uint64_t w22 = util::SafeLoadAs(in); - w22 = arrow::bit_util::FromLittleEndian(w22); - in += 4; - out[0] = (w0)&mask; - out[1] = ((w0 >> 45) | (w1 << 19)) & mask; - out[2] = ((w1 >> 26) | (w2 << 38)) & mask; - out[3] = (w2 >> 7) & mask; - out[4] = ((w2 >> 52) | (w3 << 12)) & mask; - out[5] = ((w3 >> 33) | (w4 << 31)) & mask; - out[6] = (w4 >> 14) & mask; - out[7] = ((w4 >> 59) | (w5 << 5)) & mask; - out[8] = ((w5 >> 40) | (w6 << 24)) & mask; - out[9] = ((w6 >> 21) | (w7 << 43)) & mask; - out[10] = (w7 >> 2) & mask; - out[11] = ((w7 >> 47) | (w8 << 17)) & mask; - out[12] = ((w8 >> 28) | (w9 << 36)) & mask; - out[13] = (w9 >> 9) & mask; - out[14] = ((w9 >> 54) | (w10 << 10)) & mask; - out[15] = ((w10 >> 35) | (w11 << 29)) & mask; - out[16] = (w11 >> 16) & mask; - out[17] = ((w11 >> 61) | (w12 << 3)) & mask; - out[18] = ((w12 >> 42) | (w13 << 22)) & mask; - out[19] = ((w13 >> 23) | (w14 << 41)) & mask; - out[20] = (w14 >> 4) & mask; - out[21] = ((w14 >> 49) | (w15 << 15)) & mask; - out[22] = ((w15 >> 30) | (w16 << 34)) & mask; - out[23] = (w16 >> 11) & mask; - out[24] = ((w16 >> 56) | (w17 << 8)) & mask; - out[25] = ((w17 >> 37) | (w18 << 27)) & mask; - out[26] = (w18 >> 18) & mask; - out[27] = ((w18 >> 63) | (w19 << 1)) & mask; - out[28] = ((w19 >> 44) | (w20 << 20)) & mask; - out[29] = ((w20 >> 25) | (w21 << 39)) & mask; - out[30] = (w21 >> 6) & mask; - out[31] = ((w21 >> 51) | (w22 << 13)) & mask; - - return in; -} - -inline const uint8_t* unpack46_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 70368744177663ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - uint64_t w21 = util::SafeLoadAs(in); - w21 = arrow::bit_util::FromLittleEndian(w21); - in += 8; - uint64_t w22 = util::SafeLoadAs(in); - w22 = arrow::bit_util::FromLittleEndian(w22); - in += 8; - out[0] = (w0)&mask; - out[1] = ((w0 >> 46) | (w1 << 18)) & mask; - out[2] = ((w1 >> 28) | (w2 << 36)) & mask; - out[3] = (w2 >> 10) & mask; - out[4] = ((w2 >> 56) | (w3 << 8)) & mask; - out[5] = ((w3 >> 38) | (w4 << 26)) & mask; - out[6] = ((w4 >> 20) | (w5 << 44)) & mask; - out[7] = (w5 >> 2) & mask; - out[8] = ((w5 >> 48) | (w6 << 16)) & mask; - out[9] = ((w6 >> 30) | (w7 << 34)) & mask; - out[10] = (w7 >> 12) & mask; - out[11] = ((w7 >> 58) | (w8 << 6)) & mask; - out[12] = ((w8 >> 40) | (w9 << 24)) & mask; - out[13] = ((w9 >> 22) | (w10 << 42)) & mask; - out[14] = (w10 >> 4) & mask; - out[15] = ((w10 >> 50) | (w11 << 14)) & mask; - out[16] = ((w11 >> 32) | (w12 << 32)) & mask; - out[17] = (w12 >> 14) & mask; - out[18] = ((w12 >> 60) | (w13 << 4)) & mask; - out[19] = ((w13 >> 42) | (w14 << 22)) & mask; - out[20] = ((w14 >> 24) | (w15 << 40)) & mask; - out[21] = (w15 >> 6) & mask; - out[22] = ((w15 >> 52) | (w16 << 12)) & mask; - out[23] = ((w16 >> 34) | (w17 << 30)) & mask; - out[24] = (w17 >> 16) & mask; - out[25] = ((w17 >> 62) | (w18 << 2)) & mask; - out[26] = ((w18 >> 44) | (w19 << 20)) & mask; - out[27] = ((w19 >> 26) | (w20 << 38)) & mask; - out[28] = (w20 >> 8) & mask; - out[29] = ((w20 >> 54) | (w21 << 10)) & mask; - out[30] = ((w21 >> 36) | (w22 << 28)) & mask; - out[31] = w22 >> 18; - - return in; -} - -inline const uint8_t* unpack47_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 140737488355327ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - uint64_t w21 = util::SafeLoadAs(in); - w21 = arrow::bit_util::FromLittleEndian(w21); - in += 8; - uint64_t w22 = util::SafeLoadAs(in); - w22 = arrow::bit_util::FromLittleEndian(w22); - in += 8; - uint64_t w23 = util::SafeLoadAs(in); - w23 = arrow::bit_util::FromLittleEndian(w23); - in += 4; - out[0] = (w0)&mask; - out[1] = ((w0 >> 47) | (w1 << 17)) & mask; - out[2] = ((w1 >> 30) | (w2 << 34)) & mask; - out[3] = (w2 >> 13) & mask; - out[4] = ((w2 >> 60) | (w3 << 4)) & mask; - out[5] = ((w3 >> 43) | (w4 << 21)) & mask; - out[6] = ((w4 >> 26) | (w5 << 38)) & mask; - out[7] = (w5 >> 9) & mask; - out[8] = ((w5 >> 56) | (w6 << 8)) & mask; - out[9] = ((w6 >> 39) | (w7 << 25)) & mask; - out[10] = ((w7 >> 22) | (w8 << 42)) & mask; - out[11] = (w8 >> 5) & mask; - out[12] = ((w8 >> 52) | (w9 << 12)) & mask; - out[13] = ((w9 >> 35) | (w10 << 29)) & mask; - out[14] = ((w10 >> 18) | (w11 << 46)) & mask; - out[15] = (w11 >> 1) & mask; - out[16] = ((w11 >> 48) | (w12 << 16)) & mask; - out[17] = ((w12 >> 31) | (w13 << 33)) & mask; - out[18] = (w13 >> 14) & mask; - out[19] = ((w13 >> 61) | (w14 << 3)) & mask; - out[20] = ((w14 >> 44) | (w15 << 20)) & mask; - out[21] = ((w15 >> 27) | (w16 << 37)) & mask; - out[22] = (w16 >> 10) & mask; - out[23] = ((w16 >> 57) | (w17 << 7)) & mask; - out[24] = ((w17 >> 40) | (w18 << 24)) & mask; - out[25] = ((w18 >> 23) | (w19 << 41)) & mask; - out[26] = (w19 >> 6) & mask; - out[27] = ((w19 >> 53) | (w20 << 11)) & mask; - out[28] = ((w20 >> 36) | (w21 << 28)) & mask; - out[29] = ((w21 >> 19) | (w22 << 45)) & mask; - out[30] = (w22 >> 2) & mask; - out[31] = ((w22 >> 49) | (w23 << 15)) & mask; - - return in; -} - -inline const uint8_t* unpack48_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 281474976710655ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - uint64_t w21 = util::SafeLoadAs(in); - w21 = arrow::bit_util::FromLittleEndian(w21); - in += 8; - uint64_t w22 = util::SafeLoadAs(in); - w22 = arrow::bit_util::FromLittleEndian(w22); - in += 8; - uint64_t w23 = util::SafeLoadAs(in); - w23 = arrow::bit_util::FromLittleEndian(w23); - in += 8; - out[0] = (w0)&mask; - out[1] = ((w0 >> 48) | (w1 << 16)) & mask; - out[2] = ((w1 >> 32) | (w2 << 32)) & mask; - out[3] = w2 >> 16; - out[4] = (w3)&mask; - out[5] = ((w3 >> 48) | (w4 << 16)) & mask; - out[6] = ((w4 >> 32) | (w5 << 32)) & mask; - out[7] = w5 >> 16; - out[8] = (w6)&mask; - out[9] = ((w6 >> 48) | (w7 << 16)) & mask; - out[10] = ((w7 >> 32) | (w8 << 32)) & mask; - out[11] = w8 >> 16; - out[12] = (w9)&mask; - out[13] = ((w9 >> 48) | (w10 << 16)) & mask; - out[14] = ((w10 >> 32) | (w11 << 32)) & mask; - out[15] = w11 >> 16; - out[16] = (w12)&mask; - out[17] = ((w12 >> 48) | (w13 << 16)) & mask; - out[18] = ((w13 >> 32) | (w14 << 32)) & mask; - out[19] = w14 >> 16; - out[20] = (w15)&mask; - out[21] = ((w15 >> 48) | (w16 << 16)) & mask; - out[22] = ((w16 >> 32) | (w17 << 32)) & mask; - out[23] = w17 >> 16; - out[24] = (w18)&mask; - out[25] = ((w18 >> 48) | (w19 << 16)) & mask; - out[26] = ((w19 >> 32) | (w20 << 32)) & mask; - out[27] = w20 >> 16; - out[28] = (w21)&mask; - out[29] = ((w21 >> 48) | (w22 << 16)) & mask; - out[30] = ((w22 >> 32) | (w23 << 32)) & mask; - out[31] = w23 >> 16; - - return in; -} - -inline const uint8_t* unpack49_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 562949953421311ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - uint64_t w21 = util::SafeLoadAs(in); - w21 = arrow::bit_util::FromLittleEndian(w21); - in += 8; - uint64_t w22 = util::SafeLoadAs(in); - w22 = arrow::bit_util::FromLittleEndian(w22); - in += 8; - uint64_t w23 = util::SafeLoadAs(in); - w23 = arrow::bit_util::FromLittleEndian(w23); - in += 8; - uint64_t w24 = util::SafeLoadAs(in); - w24 = arrow::bit_util::FromLittleEndian(w24); - in += 4; - out[0] = (w0)&mask; - out[1] = ((w0 >> 49) | (w1 << 15)) & mask; - out[2] = ((w1 >> 34) | (w2 << 30)) & mask; - out[3] = ((w2 >> 19) | (w3 << 45)) & mask; - out[4] = (w3 >> 4) & mask; - out[5] = ((w3 >> 53) | (w4 << 11)) & mask; - out[6] = ((w4 >> 38) | (w5 << 26)) & mask; - out[7] = ((w5 >> 23) | (w6 << 41)) & mask; - out[8] = (w6 >> 8) & mask; - out[9] = ((w6 >> 57) | (w7 << 7)) & mask; - out[10] = ((w7 >> 42) | (w8 << 22)) & mask; - out[11] = ((w8 >> 27) | (w9 << 37)) & mask; - out[12] = (w9 >> 12) & mask; - out[13] = ((w9 >> 61) | (w10 << 3)) & mask; - out[14] = ((w10 >> 46) | (w11 << 18)) & mask; - out[15] = ((w11 >> 31) | (w12 << 33)) & mask; - out[16] = ((w12 >> 16) | (w13 << 48)) & mask; - out[17] = (w13 >> 1) & mask; - out[18] = ((w13 >> 50) | (w14 << 14)) & mask; - out[19] = ((w14 >> 35) | (w15 << 29)) & mask; - out[20] = ((w15 >> 20) | (w16 << 44)) & mask; - out[21] = (w16 >> 5) & mask; - out[22] = ((w16 >> 54) | (w17 << 10)) & mask; - out[23] = ((w17 >> 39) | (w18 << 25)) & mask; - out[24] = ((w18 >> 24) | (w19 << 40)) & mask; - out[25] = (w19 >> 9) & mask; - out[26] = ((w19 >> 58) | (w20 << 6)) & mask; - out[27] = ((w20 >> 43) | (w21 << 21)) & mask; - out[28] = ((w21 >> 28) | (w22 << 36)) & mask; - out[29] = (w22 >> 13) & mask; - out[30] = ((w22 >> 62) | (w23 << 2)) & mask; - out[31] = ((w23 >> 47) | (w24 << 17)) & mask; - - return in; -} - -inline const uint8_t* unpack50_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 1125899906842623ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - uint64_t w21 = util::SafeLoadAs(in); - w21 = arrow::bit_util::FromLittleEndian(w21); - in += 8; - uint64_t w22 = util::SafeLoadAs(in); - w22 = arrow::bit_util::FromLittleEndian(w22); - in += 8; - uint64_t w23 = util::SafeLoadAs(in); - w23 = arrow::bit_util::FromLittleEndian(w23); - in += 8; - uint64_t w24 = util::SafeLoadAs(in); - w24 = arrow::bit_util::FromLittleEndian(w24); - in += 8; - out[0] = (w0)&mask; - out[1] = ((w0 >> 50) | (w1 << 14)) & mask; - out[2] = ((w1 >> 36) | (w2 << 28)) & mask; - out[3] = ((w2 >> 22) | (w3 << 42)) & mask; - out[4] = (w3 >> 8) & mask; - out[5] = ((w3 >> 58) | (w4 << 6)) & mask; - out[6] = ((w4 >> 44) | (w5 << 20)) & mask; - out[7] = ((w5 >> 30) | (w6 << 34)) & mask; - out[8] = ((w6 >> 16) | (w7 << 48)) & mask; - out[9] = (w7 >> 2) & mask; - out[10] = ((w7 >> 52) | (w8 << 12)) & mask; - out[11] = ((w8 >> 38) | (w9 << 26)) & mask; - out[12] = ((w9 >> 24) | (w10 << 40)) & mask; - out[13] = (w10 >> 10) & mask; - out[14] = ((w10 >> 60) | (w11 << 4)) & mask; - out[15] = ((w11 >> 46) | (w12 << 18)) & mask; - out[16] = ((w12 >> 32) | (w13 << 32)) & mask; - out[17] = ((w13 >> 18) | (w14 << 46)) & mask; - out[18] = (w14 >> 4) & mask; - out[19] = ((w14 >> 54) | (w15 << 10)) & mask; - out[20] = ((w15 >> 40) | (w16 << 24)) & mask; - out[21] = ((w16 >> 26) | (w17 << 38)) & mask; - out[22] = (w17 >> 12) & mask; - out[23] = ((w17 >> 62) | (w18 << 2)) & mask; - out[24] = ((w18 >> 48) | (w19 << 16)) & mask; - out[25] = ((w19 >> 34) | (w20 << 30)) & mask; - out[26] = ((w20 >> 20) | (w21 << 44)) & mask; - out[27] = (w21 >> 6) & mask; - out[28] = ((w21 >> 56) | (w22 << 8)) & mask; - out[29] = ((w22 >> 42) | (w23 << 22)) & mask; - out[30] = ((w23 >> 28) | (w24 << 36)) & mask; - out[31] = w24 >> 14; - - return in; -} - -inline const uint8_t* unpack51_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 2251799813685247ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - uint64_t w21 = util::SafeLoadAs(in); - w21 = arrow::bit_util::FromLittleEndian(w21); - in += 8; - uint64_t w22 = util::SafeLoadAs(in); - w22 = arrow::bit_util::FromLittleEndian(w22); - in += 8; - uint64_t w23 = util::SafeLoadAs(in); - w23 = arrow::bit_util::FromLittleEndian(w23); - in += 8; - uint64_t w24 = util::SafeLoadAs(in); - w24 = arrow::bit_util::FromLittleEndian(w24); - in += 8; - uint64_t w25 = util::SafeLoadAs(in); - w25 = arrow::bit_util::FromLittleEndian(w25); - in += 4; - out[0] = (w0)&mask; - out[1] = ((w0 >> 51) | (w1 << 13)) & mask; - out[2] = ((w1 >> 38) | (w2 << 26)) & mask; - out[3] = ((w2 >> 25) | (w3 << 39)) & mask; - out[4] = (w3 >> 12) & mask; - out[5] = ((w3 >> 63) | (w4 << 1)) & mask; - out[6] = ((w4 >> 50) | (w5 << 14)) & mask; - out[7] = ((w5 >> 37) | (w6 << 27)) & mask; - out[8] = ((w6 >> 24) | (w7 << 40)) & mask; - out[9] = (w7 >> 11) & mask; - out[10] = ((w7 >> 62) | (w8 << 2)) & mask; - out[11] = ((w8 >> 49) | (w9 << 15)) & mask; - out[12] = ((w9 >> 36) | (w10 << 28)) & mask; - out[13] = ((w10 >> 23) | (w11 << 41)) & mask; - out[14] = (w11 >> 10) & mask; - out[15] = ((w11 >> 61) | (w12 << 3)) & mask; - out[16] = ((w12 >> 48) | (w13 << 16)) & mask; - out[17] = ((w13 >> 35) | (w14 << 29)) & mask; - out[18] = ((w14 >> 22) | (w15 << 42)) & mask; - out[19] = (w15 >> 9) & mask; - out[20] = ((w15 >> 60) | (w16 << 4)) & mask; - out[21] = ((w16 >> 47) | (w17 << 17)) & mask; - out[22] = ((w17 >> 34) | (w18 << 30)) & mask; - out[23] = ((w18 >> 21) | (w19 << 43)) & mask; - out[24] = (w19 >> 8) & mask; - out[25] = ((w19 >> 59) | (w20 << 5)) & mask; - out[26] = ((w20 >> 46) | (w21 << 18)) & mask; - out[27] = ((w21 >> 33) | (w22 << 31)) & mask; - out[28] = ((w22 >> 20) | (w23 << 44)) & mask; - out[29] = (w23 >> 7) & mask; - out[30] = ((w23 >> 58) | (w24 << 6)) & mask; - out[31] = ((w24 >> 45) | (w25 << 19)) & mask; - - return in; -} - -inline const uint8_t* unpack52_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 4503599627370495ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - uint64_t w21 = util::SafeLoadAs(in); - w21 = arrow::bit_util::FromLittleEndian(w21); - in += 8; - uint64_t w22 = util::SafeLoadAs(in); - w22 = arrow::bit_util::FromLittleEndian(w22); - in += 8; - uint64_t w23 = util::SafeLoadAs(in); - w23 = arrow::bit_util::FromLittleEndian(w23); - in += 8; - uint64_t w24 = util::SafeLoadAs(in); - w24 = arrow::bit_util::FromLittleEndian(w24); - in += 8; - uint64_t w25 = util::SafeLoadAs(in); - w25 = arrow::bit_util::FromLittleEndian(w25); - in += 8; - out[0] = (w0)&mask; - out[1] = ((w0 >> 52) | (w1 << 12)) & mask; - out[2] = ((w1 >> 40) | (w2 << 24)) & mask; - out[3] = ((w2 >> 28) | (w3 << 36)) & mask; - out[4] = ((w3 >> 16) | (w4 << 48)) & mask; - out[5] = (w4 >> 4) & mask; - out[6] = ((w4 >> 56) | (w5 << 8)) & mask; - out[7] = ((w5 >> 44) | (w6 << 20)) & mask; - out[8] = ((w6 >> 32) | (w7 << 32)) & mask; - out[9] = ((w7 >> 20) | (w8 << 44)) & mask; - out[10] = (w8 >> 8) & mask; - out[11] = ((w8 >> 60) | (w9 << 4)) & mask; - out[12] = ((w9 >> 48) | (w10 << 16)) & mask; - out[13] = ((w10 >> 36) | (w11 << 28)) & mask; - out[14] = ((w11 >> 24) | (w12 << 40)) & mask; - out[15] = w12 >> 12; - out[16] = (w13)&mask; - out[17] = ((w13 >> 52) | (w14 << 12)) & mask; - out[18] = ((w14 >> 40) | (w15 << 24)) & mask; - out[19] = ((w15 >> 28) | (w16 << 36)) & mask; - out[20] = ((w16 >> 16) | (w17 << 48)) & mask; - out[21] = (w17 >> 4) & mask; - out[22] = ((w17 >> 56) | (w18 << 8)) & mask; - out[23] = ((w18 >> 44) | (w19 << 20)) & mask; - out[24] = ((w19 >> 32) | (w20 << 32)) & mask; - out[25] = ((w20 >> 20) | (w21 << 44)) & mask; - out[26] = (w21 >> 8) & mask; - out[27] = ((w21 >> 60) | (w22 << 4)) & mask; - out[28] = ((w22 >> 48) | (w23 << 16)) & mask; - out[29] = ((w23 >> 36) | (w24 << 28)) & mask; - out[30] = ((w24 >> 24) | (w25 << 40)) & mask; - out[31] = w25 >> 12; - - return in; -} - -inline const uint8_t* unpack53_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 9007199254740991ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - uint64_t w21 = util::SafeLoadAs(in); - w21 = arrow::bit_util::FromLittleEndian(w21); - in += 8; - uint64_t w22 = util::SafeLoadAs(in); - w22 = arrow::bit_util::FromLittleEndian(w22); - in += 8; - uint64_t w23 = util::SafeLoadAs(in); - w23 = arrow::bit_util::FromLittleEndian(w23); - in += 8; - uint64_t w24 = util::SafeLoadAs(in); - w24 = arrow::bit_util::FromLittleEndian(w24); - in += 8; - uint64_t w25 = util::SafeLoadAs(in); - w25 = arrow::bit_util::FromLittleEndian(w25); - in += 8; - uint64_t w26 = util::SafeLoadAs(in); - w26 = arrow::bit_util::FromLittleEndian(w26); - in += 4; - out[0] = (w0)&mask; - out[1] = ((w0 >> 53) | (w1 << 11)) & mask; - out[2] = ((w1 >> 42) | (w2 << 22)) & mask; - out[3] = ((w2 >> 31) | (w3 << 33)) & mask; - out[4] = ((w3 >> 20) | (w4 << 44)) & mask; - out[5] = (w4 >> 9) & mask; - out[6] = ((w4 >> 62) | (w5 << 2)) & mask; - out[7] = ((w5 >> 51) | (w6 << 13)) & mask; - out[8] = ((w6 >> 40) | (w7 << 24)) & mask; - out[9] = ((w7 >> 29) | (w8 << 35)) & mask; - out[10] = ((w8 >> 18) | (w9 << 46)) & mask; - out[11] = (w9 >> 7) & mask; - out[12] = ((w9 >> 60) | (w10 << 4)) & mask; - out[13] = ((w10 >> 49) | (w11 << 15)) & mask; - out[14] = ((w11 >> 38) | (w12 << 26)) & mask; - out[15] = ((w12 >> 27) | (w13 << 37)) & mask; - out[16] = ((w13 >> 16) | (w14 << 48)) & mask; - out[17] = (w14 >> 5) & mask; - out[18] = ((w14 >> 58) | (w15 << 6)) & mask; - out[19] = ((w15 >> 47) | (w16 << 17)) & mask; - out[20] = ((w16 >> 36) | (w17 << 28)) & mask; - out[21] = ((w17 >> 25) | (w18 << 39)) & mask; - out[22] = ((w18 >> 14) | (w19 << 50)) & mask; - out[23] = (w19 >> 3) & mask; - out[24] = ((w19 >> 56) | (w20 << 8)) & mask; - out[25] = ((w20 >> 45) | (w21 << 19)) & mask; - out[26] = ((w21 >> 34) | (w22 << 30)) & mask; - out[27] = ((w22 >> 23) | (w23 << 41)) & mask; - out[28] = ((w23 >> 12) | (w24 << 52)) & mask; - out[29] = (w24 >> 1) & mask; - out[30] = ((w24 >> 54) | (w25 << 10)) & mask; - out[31] = ((w25 >> 43) | (w26 << 21)) & mask; - - return in; -} - -inline const uint8_t* unpack54_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 18014398509481983ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - uint64_t w21 = util::SafeLoadAs(in); - w21 = arrow::bit_util::FromLittleEndian(w21); - in += 8; - uint64_t w22 = util::SafeLoadAs(in); - w22 = arrow::bit_util::FromLittleEndian(w22); - in += 8; - uint64_t w23 = util::SafeLoadAs(in); - w23 = arrow::bit_util::FromLittleEndian(w23); - in += 8; - uint64_t w24 = util::SafeLoadAs(in); - w24 = arrow::bit_util::FromLittleEndian(w24); - in += 8; - uint64_t w25 = util::SafeLoadAs(in); - w25 = arrow::bit_util::FromLittleEndian(w25); - in += 8; - uint64_t w26 = util::SafeLoadAs(in); - w26 = arrow::bit_util::FromLittleEndian(w26); - in += 8; - out[0] = (w0)&mask; - out[1] = ((w0 >> 54) | (w1 << 10)) & mask; - out[2] = ((w1 >> 44) | (w2 << 20)) & mask; - out[3] = ((w2 >> 34) | (w3 << 30)) & mask; - out[4] = ((w3 >> 24) | (w4 << 40)) & mask; - out[5] = ((w4 >> 14) | (w5 << 50)) & mask; - out[6] = (w5 >> 4) & mask; - out[7] = ((w5 >> 58) | (w6 << 6)) & mask; - out[8] = ((w6 >> 48) | (w7 << 16)) & mask; - out[9] = ((w7 >> 38) | (w8 << 26)) & mask; - out[10] = ((w8 >> 28) | (w9 << 36)) & mask; - out[11] = ((w9 >> 18) | (w10 << 46)) & mask; - out[12] = (w10 >> 8) & mask; - out[13] = ((w10 >> 62) | (w11 << 2)) & mask; - out[14] = ((w11 >> 52) | (w12 << 12)) & mask; - out[15] = ((w12 >> 42) | (w13 << 22)) & mask; - out[16] = ((w13 >> 32) | (w14 << 32)) & mask; - out[17] = ((w14 >> 22) | (w15 << 42)) & mask; - out[18] = ((w15 >> 12) | (w16 << 52)) & mask; - out[19] = (w16 >> 2) & mask; - out[20] = ((w16 >> 56) | (w17 << 8)) & mask; - out[21] = ((w17 >> 46) | (w18 << 18)) & mask; - out[22] = ((w18 >> 36) | (w19 << 28)) & mask; - out[23] = ((w19 >> 26) | (w20 << 38)) & mask; - out[24] = ((w20 >> 16) | (w21 << 48)) & mask; - out[25] = (w21 >> 6) & mask; - out[26] = ((w21 >> 60) | (w22 << 4)) & mask; - out[27] = ((w22 >> 50) | (w23 << 14)) & mask; - out[28] = ((w23 >> 40) | (w24 << 24)) & mask; - out[29] = ((w24 >> 30) | (w25 << 34)) & mask; - out[30] = ((w25 >> 20) | (w26 << 44)) & mask; - out[31] = w26 >> 10; - - return in; -} - -inline const uint8_t* unpack55_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 36028797018963967ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - uint64_t w21 = util::SafeLoadAs(in); - w21 = arrow::bit_util::FromLittleEndian(w21); - in += 8; - uint64_t w22 = util::SafeLoadAs(in); - w22 = arrow::bit_util::FromLittleEndian(w22); - in += 8; - uint64_t w23 = util::SafeLoadAs(in); - w23 = arrow::bit_util::FromLittleEndian(w23); - in += 8; - uint64_t w24 = util::SafeLoadAs(in); - w24 = arrow::bit_util::FromLittleEndian(w24); - in += 8; - uint64_t w25 = util::SafeLoadAs(in); - w25 = arrow::bit_util::FromLittleEndian(w25); - in += 8; - uint64_t w26 = util::SafeLoadAs(in); - w26 = arrow::bit_util::FromLittleEndian(w26); - in += 8; - uint64_t w27 = util::SafeLoadAs(in); - w27 = arrow::bit_util::FromLittleEndian(w27); - in += 4; - out[0] = (w0)&mask; - out[1] = ((w0 >> 55) | (w1 << 9)) & mask; - out[2] = ((w1 >> 46) | (w2 << 18)) & mask; - out[3] = ((w2 >> 37) | (w3 << 27)) & mask; - out[4] = ((w3 >> 28) | (w4 << 36)) & mask; - out[5] = ((w4 >> 19) | (w5 << 45)) & mask; - out[6] = ((w5 >> 10) | (w6 << 54)) & mask; - out[7] = (w6 >> 1) & mask; - out[8] = ((w6 >> 56) | (w7 << 8)) & mask; - out[9] = ((w7 >> 47) | (w8 << 17)) & mask; - out[10] = ((w8 >> 38) | (w9 << 26)) & mask; - out[11] = ((w9 >> 29) | (w10 << 35)) & mask; - out[12] = ((w10 >> 20) | (w11 << 44)) & mask; - out[13] = ((w11 >> 11) | (w12 << 53)) & mask; - out[14] = (w12 >> 2) & mask; - out[15] = ((w12 >> 57) | (w13 << 7)) & mask; - out[16] = ((w13 >> 48) | (w14 << 16)) & mask; - out[17] = ((w14 >> 39) | (w15 << 25)) & mask; - out[18] = ((w15 >> 30) | (w16 << 34)) & mask; - out[19] = ((w16 >> 21) | (w17 << 43)) & mask; - out[20] = ((w17 >> 12) | (w18 << 52)) & mask; - out[21] = (w18 >> 3) & mask; - out[22] = ((w18 >> 58) | (w19 << 6)) & mask; - out[23] = ((w19 >> 49) | (w20 << 15)) & mask; - out[24] = ((w20 >> 40) | (w21 << 24)) & mask; - out[25] = ((w21 >> 31) | (w22 << 33)) & mask; - out[26] = ((w22 >> 22) | (w23 << 42)) & mask; - out[27] = ((w23 >> 13) | (w24 << 51)) & mask; - out[28] = (w24 >> 4) & mask; - out[29] = ((w24 >> 59) | (w25 << 5)) & mask; - out[30] = ((w25 >> 50) | (w26 << 14)) & mask; - out[31] = ((w26 >> 41) | (w27 << 23)) & mask; - - return in; -} - -inline const uint8_t* unpack56_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 72057594037927935ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - uint64_t w21 = util::SafeLoadAs(in); - w21 = arrow::bit_util::FromLittleEndian(w21); - in += 8; - uint64_t w22 = util::SafeLoadAs(in); - w22 = arrow::bit_util::FromLittleEndian(w22); - in += 8; - uint64_t w23 = util::SafeLoadAs(in); - w23 = arrow::bit_util::FromLittleEndian(w23); - in += 8; - uint64_t w24 = util::SafeLoadAs(in); - w24 = arrow::bit_util::FromLittleEndian(w24); - in += 8; - uint64_t w25 = util::SafeLoadAs(in); - w25 = arrow::bit_util::FromLittleEndian(w25); - in += 8; - uint64_t w26 = util::SafeLoadAs(in); - w26 = arrow::bit_util::FromLittleEndian(w26); - in += 8; - uint64_t w27 = util::SafeLoadAs(in); - w27 = arrow::bit_util::FromLittleEndian(w27); - in += 8; - out[0] = (w0)&mask; - out[1] = ((w0 >> 56) | (w1 << 8)) & mask; - out[2] = ((w1 >> 48) | (w2 << 16)) & mask; - out[3] = ((w2 >> 40) | (w3 << 24)) & mask; - out[4] = ((w3 >> 32) | (w4 << 32)) & mask; - out[5] = ((w4 >> 24) | (w5 << 40)) & mask; - out[6] = ((w5 >> 16) | (w6 << 48)) & mask; - out[7] = w6 >> 8; - out[8] = (w7)&mask; - out[9] = ((w7 >> 56) | (w8 << 8)) & mask; - out[10] = ((w8 >> 48) | (w9 << 16)) & mask; - out[11] = ((w9 >> 40) | (w10 << 24)) & mask; - out[12] = ((w10 >> 32) | (w11 << 32)) & mask; - out[13] = ((w11 >> 24) | (w12 << 40)) & mask; - out[14] = ((w12 >> 16) | (w13 << 48)) & mask; - out[15] = w13 >> 8; - out[16] = (w14)&mask; - out[17] = ((w14 >> 56) | (w15 << 8)) & mask; - out[18] = ((w15 >> 48) | (w16 << 16)) & mask; - out[19] = ((w16 >> 40) | (w17 << 24)) & mask; - out[20] = ((w17 >> 32) | (w18 << 32)) & mask; - out[21] = ((w18 >> 24) | (w19 << 40)) & mask; - out[22] = ((w19 >> 16) | (w20 << 48)) & mask; - out[23] = w20 >> 8; - out[24] = (w21)&mask; - out[25] = ((w21 >> 56) | (w22 << 8)) & mask; - out[26] = ((w22 >> 48) | (w23 << 16)) & mask; - out[27] = ((w23 >> 40) | (w24 << 24)) & mask; - out[28] = ((w24 >> 32) | (w25 << 32)) & mask; - out[29] = ((w25 >> 24) | (w26 << 40)) & mask; - out[30] = ((w26 >> 16) | (w27 << 48)) & mask; - out[31] = w27 >> 8; - - return in; -} - -inline const uint8_t* unpack57_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 144115188075855871ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - uint64_t w21 = util::SafeLoadAs(in); - w21 = arrow::bit_util::FromLittleEndian(w21); - in += 8; - uint64_t w22 = util::SafeLoadAs(in); - w22 = arrow::bit_util::FromLittleEndian(w22); - in += 8; - uint64_t w23 = util::SafeLoadAs(in); - w23 = arrow::bit_util::FromLittleEndian(w23); - in += 8; - uint64_t w24 = util::SafeLoadAs(in); - w24 = arrow::bit_util::FromLittleEndian(w24); - in += 8; - uint64_t w25 = util::SafeLoadAs(in); - w25 = arrow::bit_util::FromLittleEndian(w25); - in += 8; - uint64_t w26 = util::SafeLoadAs(in); - w26 = arrow::bit_util::FromLittleEndian(w26); - in += 8; - uint64_t w27 = util::SafeLoadAs(in); - w27 = arrow::bit_util::FromLittleEndian(w27); - in += 8; - uint64_t w28 = util::SafeLoadAs(in); - w28 = arrow::bit_util::FromLittleEndian(w28); - in += 4; - out[0] = (w0)&mask; - out[1] = ((w0 >> 57) | (w1 << 7)) & mask; - out[2] = ((w1 >> 50) | (w2 << 14)) & mask; - out[3] = ((w2 >> 43) | (w3 << 21)) & mask; - out[4] = ((w3 >> 36) | (w4 << 28)) & mask; - out[5] = ((w4 >> 29) | (w5 << 35)) & mask; - out[6] = ((w5 >> 22) | (w6 << 42)) & mask; - out[7] = ((w6 >> 15) | (w7 << 49)) & mask; - out[8] = ((w7 >> 8) | (w8 << 56)) & mask; - out[9] = (w8 >> 1) & mask; - out[10] = ((w8 >> 58) | (w9 << 6)) & mask; - out[11] = ((w9 >> 51) | (w10 << 13)) & mask; - out[12] = ((w10 >> 44) | (w11 << 20)) & mask; - out[13] = ((w11 >> 37) | (w12 << 27)) & mask; - out[14] = ((w12 >> 30) | (w13 << 34)) & mask; - out[15] = ((w13 >> 23) | (w14 << 41)) & mask; - out[16] = ((w14 >> 16) | (w15 << 48)) & mask; - out[17] = ((w15 >> 9) | (w16 << 55)) & mask; - out[18] = (w16 >> 2) & mask; - out[19] = ((w16 >> 59) | (w17 << 5)) & mask; - out[20] = ((w17 >> 52) | (w18 << 12)) & mask; - out[21] = ((w18 >> 45) | (w19 << 19)) & mask; - out[22] = ((w19 >> 38) | (w20 << 26)) & mask; - out[23] = ((w20 >> 31) | (w21 << 33)) & mask; - out[24] = ((w21 >> 24) | (w22 << 40)) & mask; - out[25] = ((w22 >> 17) | (w23 << 47)) & mask; - out[26] = ((w23 >> 10) | (w24 << 54)) & mask; - out[27] = (w24 >> 3) & mask; - out[28] = ((w24 >> 60) | (w25 << 4)) & mask; - out[29] = ((w25 >> 53) | (w26 << 11)) & mask; - out[30] = ((w26 >> 46) | (w27 << 18)) & mask; - out[31] = ((w27 >> 39) | (w28 << 25)) & mask; - - return in; -} - -inline const uint8_t* unpack58_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 288230376151711743ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - uint64_t w21 = util::SafeLoadAs(in); - w21 = arrow::bit_util::FromLittleEndian(w21); - in += 8; - uint64_t w22 = util::SafeLoadAs(in); - w22 = arrow::bit_util::FromLittleEndian(w22); - in += 8; - uint64_t w23 = util::SafeLoadAs(in); - w23 = arrow::bit_util::FromLittleEndian(w23); - in += 8; - uint64_t w24 = util::SafeLoadAs(in); - w24 = arrow::bit_util::FromLittleEndian(w24); - in += 8; - uint64_t w25 = util::SafeLoadAs(in); - w25 = arrow::bit_util::FromLittleEndian(w25); - in += 8; - uint64_t w26 = util::SafeLoadAs(in); - w26 = arrow::bit_util::FromLittleEndian(w26); - in += 8; - uint64_t w27 = util::SafeLoadAs(in); - w27 = arrow::bit_util::FromLittleEndian(w27); - in += 8; - uint64_t w28 = util::SafeLoadAs(in); - w28 = arrow::bit_util::FromLittleEndian(w28); - in += 8; - out[0] = (w0)&mask; - out[1] = ((w0 >> 58) | (w1 << 6)) & mask; - out[2] = ((w1 >> 52) | (w2 << 12)) & mask; - out[3] = ((w2 >> 46) | (w3 << 18)) & mask; - out[4] = ((w3 >> 40) | (w4 << 24)) & mask; - out[5] = ((w4 >> 34) | (w5 << 30)) & mask; - out[6] = ((w5 >> 28) | (w6 << 36)) & mask; - out[7] = ((w6 >> 22) | (w7 << 42)) & mask; - out[8] = ((w7 >> 16) | (w8 << 48)) & mask; - out[9] = ((w8 >> 10) | (w9 << 54)) & mask; - out[10] = (w9 >> 4) & mask; - out[11] = ((w9 >> 62) | (w10 << 2)) & mask; - out[12] = ((w10 >> 56) | (w11 << 8)) & mask; - out[13] = ((w11 >> 50) | (w12 << 14)) & mask; - out[14] = ((w12 >> 44) | (w13 << 20)) & mask; - out[15] = ((w13 >> 38) | (w14 << 26)) & mask; - out[16] = ((w14 >> 32) | (w15 << 32)) & mask; - out[17] = ((w15 >> 26) | (w16 << 38)) & mask; - out[18] = ((w16 >> 20) | (w17 << 44)) & mask; - out[19] = ((w17 >> 14) | (w18 << 50)) & mask; - out[20] = ((w18 >> 8) | (w19 << 56)) & mask; - out[21] = (w19 >> 2) & mask; - out[22] = ((w19 >> 60) | (w20 << 4)) & mask; - out[23] = ((w20 >> 54) | (w21 << 10)) & mask; - out[24] = ((w21 >> 48) | (w22 << 16)) & mask; - out[25] = ((w22 >> 42) | (w23 << 22)) & mask; - out[26] = ((w23 >> 36) | (w24 << 28)) & mask; - out[27] = ((w24 >> 30) | (w25 << 34)) & mask; - out[28] = ((w25 >> 24) | (w26 << 40)) & mask; - out[29] = ((w26 >> 18) | (w27 << 46)) & mask; - out[30] = ((w27 >> 12) | (w28 << 52)) & mask; - out[31] = w28 >> 6; - - return in; -} - -inline const uint8_t* unpack59_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 576460752303423487ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - uint64_t w21 = util::SafeLoadAs(in); - w21 = arrow::bit_util::FromLittleEndian(w21); - in += 8; - uint64_t w22 = util::SafeLoadAs(in); - w22 = arrow::bit_util::FromLittleEndian(w22); - in += 8; - uint64_t w23 = util::SafeLoadAs(in); - w23 = arrow::bit_util::FromLittleEndian(w23); - in += 8; - uint64_t w24 = util::SafeLoadAs(in); - w24 = arrow::bit_util::FromLittleEndian(w24); - in += 8; - uint64_t w25 = util::SafeLoadAs(in); - w25 = arrow::bit_util::FromLittleEndian(w25); - in += 8; - uint64_t w26 = util::SafeLoadAs(in); - w26 = arrow::bit_util::FromLittleEndian(w26); - in += 8; - uint64_t w27 = util::SafeLoadAs(in); - w27 = arrow::bit_util::FromLittleEndian(w27); - in += 8; - uint64_t w28 = util::SafeLoadAs(in); - w28 = arrow::bit_util::FromLittleEndian(w28); - in += 8; - uint64_t w29 = util::SafeLoadAs(in); - w29 = arrow::bit_util::FromLittleEndian(w29); - in += 4; - out[0] = (w0)&mask; - out[1] = ((w0 >> 59) | (w1 << 5)) & mask; - out[2] = ((w1 >> 54) | (w2 << 10)) & mask; - out[3] = ((w2 >> 49) | (w3 << 15)) & mask; - out[4] = ((w3 >> 44) | (w4 << 20)) & mask; - out[5] = ((w4 >> 39) | (w5 << 25)) & mask; - out[6] = ((w5 >> 34) | (w6 << 30)) & mask; - out[7] = ((w6 >> 29) | (w7 << 35)) & mask; - out[8] = ((w7 >> 24) | (w8 << 40)) & mask; - out[9] = ((w8 >> 19) | (w9 << 45)) & mask; - out[10] = ((w9 >> 14) | (w10 << 50)) & mask; - out[11] = ((w10 >> 9) | (w11 << 55)) & mask; - out[12] = (w11 >> 4) & mask; - out[13] = ((w11 >> 63) | (w12 << 1)) & mask; - out[14] = ((w12 >> 58) | (w13 << 6)) & mask; - out[15] = ((w13 >> 53) | (w14 << 11)) & mask; - out[16] = ((w14 >> 48) | (w15 << 16)) & mask; - out[17] = ((w15 >> 43) | (w16 << 21)) & mask; - out[18] = ((w16 >> 38) | (w17 << 26)) & mask; - out[19] = ((w17 >> 33) | (w18 << 31)) & mask; - out[20] = ((w18 >> 28) | (w19 << 36)) & mask; - out[21] = ((w19 >> 23) | (w20 << 41)) & mask; - out[22] = ((w20 >> 18) | (w21 << 46)) & mask; - out[23] = ((w21 >> 13) | (w22 << 51)) & mask; - out[24] = ((w22 >> 8) | (w23 << 56)) & mask; - out[25] = (w23 >> 3) & mask; - out[26] = ((w23 >> 62) | (w24 << 2)) & mask; - out[27] = ((w24 >> 57) | (w25 << 7)) & mask; - out[28] = ((w25 >> 52) | (w26 << 12)) & mask; - out[29] = ((w26 >> 47) | (w27 << 17)) & mask; - out[30] = ((w27 >> 42) | (w28 << 22)) & mask; - out[31] = ((w28 >> 37) | (w29 << 27)) & mask; - - return in; -} - -inline const uint8_t* unpack60_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 1152921504606846975ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - uint64_t w21 = util::SafeLoadAs(in); - w21 = arrow::bit_util::FromLittleEndian(w21); - in += 8; - uint64_t w22 = util::SafeLoadAs(in); - w22 = arrow::bit_util::FromLittleEndian(w22); - in += 8; - uint64_t w23 = util::SafeLoadAs(in); - w23 = arrow::bit_util::FromLittleEndian(w23); - in += 8; - uint64_t w24 = util::SafeLoadAs(in); - w24 = arrow::bit_util::FromLittleEndian(w24); - in += 8; - uint64_t w25 = util::SafeLoadAs(in); - w25 = arrow::bit_util::FromLittleEndian(w25); - in += 8; - uint64_t w26 = util::SafeLoadAs(in); - w26 = arrow::bit_util::FromLittleEndian(w26); - in += 8; - uint64_t w27 = util::SafeLoadAs(in); - w27 = arrow::bit_util::FromLittleEndian(w27); - in += 8; - uint64_t w28 = util::SafeLoadAs(in); - w28 = arrow::bit_util::FromLittleEndian(w28); - in += 8; - uint64_t w29 = util::SafeLoadAs(in); - w29 = arrow::bit_util::FromLittleEndian(w29); - in += 8; - out[0] = (w0)&mask; - out[1] = ((w0 >> 60) | (w1 << 4)) & mask; - out[2] = ((w1 >> 56) | (w2 << 8)) & mask; - out[3] = ((w2 >> 52) | (w3 << 12)) & mask; - out[4] = ((w3 >> 48) | (w4 << 16)) & mask; - out[5] = ((w4 >> 44) | (w5 << 20)) & mask; - out[6] = ((w5 >> 40) | (w6 << 24)) & mask; - out[7] = ((w6 >> 36) | (w7 << 28)) & mask; - out[8] = ((w7 >> 32) | (w8 << 32)) & mask; - out[9] = ((w8 >> 28) | (w9 << 36)) & mask; - out[10] = ((w9 >> 24) | (w10 << 40)) & mask; - out[11] = ((w10 >> 20) | (w11 << 44)) & mask; - out[12] = ((w11 >> 16) | (w12 << 48)) & mask; - out[13] = ((w12 >> 12) | (w13 << 52)) & mask; - out[14] = ((w13 >> 8) | (w14 << 56)) & mask; - out[15] = w14 >> 4; - out[16] = (w15)&mask; - out[17] = ((w15 >> 60) | (w16 << 4)) & mask; - out[18] = ((w16 >> 56) | (w17 << 8)) & mask; - out[19] = ((w17 >> 52) | (w18 << 12)) & mask; - out[20] = ((w18 >> 48) | (w19 << 16)) & mask; - out[21] = ((w19 >> 44) | (w20 << 20)) & mask; - out[22] = ((w20 >> 40) | (w21 << 24)) & mask; - out[23] = ((w21 >> 36) | (w22 << 28)) & mask; - out[24] = ((w22 >> 32) | (w23 << 32)) & mask; - out[25] = ((w23 >> 28) | (w24 << 36)) & mask; - out[26] = ((w24 >> 24) | (w25 << 40)) & mask; - out[27] = ((w25 >> 20) | (w26 << 44)) & mask; - out[28] = ((w26 >> 16) | (w27 << 48)) & mask; - out[29] = ((w27 >> 12) | (w28 << 52)) & mask; - out[30] = ((w28 >> 8) | (w29 << 56)) & mask; - out[31] = w29 >> 4; - - return in; -} - -inline const uint8_t* unpack61_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 2305843009213693951ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - uint64_t w21 = util::SafeLoadAs(in); - w21 = arrow::bit_util::FromLittleEndian(w21); - in += 8; - uint64_t w22 = util::SafeLoadAs(in); - w22 = arrow::bit_util::FromLittleEndian(w22); - in += 8; - uint64_t w23 = util::SafeLoadAs(in); - w23 = arrow::bit_util::FromLittleEndian(w23); - in += 8; - uint64_t w24 = util::SafeLoadAs(in); - w24 = arrow::bit_util::FromLittleEndian(w24); - in += 8; - uint64_t w25 = util::SafeLoadAs(in); - w25 = arrow::bit_util::FromLittleEndian(w25); - in += 8; - uint64_t w26 = util::SafeLoadAs(in); - w26 = arrow::bit_util::FromLittleEndian(w26); - in += 8; - uint64_t w27 = util::SafeLoadAs(in); - w27 = arrow::bit_util::FromLittleEndian(w27); - in += 8; - uint64_t w28 = util::SafeLoadAs(in); - w28 = arrow::bit_util::FromLittleEndian(w28); - in += 8; - uint64_t w29 = util::SafeLoadAs(in); - w29 = arrow::bit_util::FromLittleEndian(w29); - in += 8; - uint64_t w30 = util::SafeLoadAs(in); - w30 = arrow::bit_util::FromLittleEndian(w30); - in += 4; - out[0] = (w0)&mask; - out[1] = ((w0 >> 61) | (w1 << 3)) & mask; - out[2] = ((w1 >> 58) | (w2 << 6)) & mask; - out[3] = ((w2 >> 55) | (w3 << 9)) & mask; - out[4] = ((w3 >> 52) | (w4 << 12)) & mask; - out[5] = ((w4 >> 49) | (w5 << 15)) & mask; - out[6] = ((w5 >> 46) | (w6 << 18)) & mask; - out[7] = ((w6 >> 43) | (w7 << 21)) & mask; - out[8] = ((w7 >> 40) | (w8 << 24)) & mask; - out[9] = ((w8 >> 37) | (w9 << 27)) & mask; - out[10] = ((w9 >> 34) | (w10 << 30)) & mask; - out[11] = ((w10 >> 31) | (w11 << 33)) & mask; - out[12] = ((w11 >> 28) | (w12 << 36)) & mask; - out[13] = ((w12 >> 25) | (w13 << 39)) & mask; - out[14] = ((w13 >> 22) | (w14 << 42)) & mask; - out[15] = ((w14 >> 19) | (w15 << 45)) & mask; - out[16] = ((w15 >> 16) | (w16 << 48)) & mask; - out[17] = ((w16 >> 13) | (w17 << 51)) & mask; - out[18] = ((w17 >> 10) | (w18 << 54)) & mask; - out[19] = ((w18 >> 7) | (w19 << 57)) & mask; - out[20] = ((w19 >> 4) | (w20 << 60)) & mask; - out[21] = (w20 >> 1) & mask; - out[22] = ((w20 >> 62) | (w21 << 2)) & mask; - out[23] = ((w21 >> 59) | (w22 << 5)) & mask; - out[24] = ((w22 >> 56) | (w23 << 8)) & mask; - out[25] = ((w23 >> 53) | (w24 << 11)) & mask; - out[26] = ((w24 >> 50) | (w25 << 14)) & mask; - out[27] = ((w25 >> 47) | (w26 << 17)) & mask; - out[28] = ((w26 >> 44) | (w27 << 20)) & mask; - out[29] = ((w27 >> 41) | (w28 << 23)) & mask; - out[30] = ((w28 >> 38) | (w29 << 26)) & mask; - out[31] = ((w29 >> 35) | (w30 << 29)) & mask; - - return in; -} - -inline const uint8_t* unpack62_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 4611686018427387903ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - uint64_t w21 = util::SafeLoadAs(in); - w21 = arrow::bit_util::FromLittleEndian(w21); - in += 8; - uint64_t w22 = util::SafeLoadAs(in); - w22 = arrow::bit_util::FromLittleEndian(w22); - in += 8; - uint64_t w23 = util::SafeLoadAs(in); - w23 = arrow::bit_util::FromLittleEndian(w23); - in += 8; - uint64_t w24 = util::SafeLoadAs(in); - w24 = arrow::bit_util::FromLittleEndian(w24); - in += 8; - uint64_t w25 = util::SafeLoadAs(in); - w25 = arrow::bit_util::FromLittleEndian(w25); - in += 8; - uint64_t w26 = util::SafeLoadAs(in); - w26 = arrow::bit_util::FromLittleEndian(w26); - in += 8; - uint64_t w27 = util::SafeLoadAs(in); - w27 = arrow::bit_util::FromLittleEndian(w27); - in += 8; - uint64_t w28 = util::SafeLoadAs(in); - w28 = arrow::bit_util::FromLittleEndian(w28); - in += 8; - uint64_t w29 = util::SafeLoadAs(in); - w29 = arrow::bit_util::FromLittleEndian(w29); - in += 8; - uint64_t w30 = util::SafeLoadAs(in); - w30 = arrow::bit_util::FromLittleEndian(w30); - in += 8; - out[0] = (w0)&mask; - out[1] = ((w0 >> 62) | (w1 << 2)) & mask; - out[2] = ((w1 >> 60) | (w2 << 4)) & mask; - out[3] = ((w2 >> 58) | (w3 << 6)) & mask; - out[4] = ((w3 >> 56) | (w4 << 8)) & mask; - out[5] = ((w4 >> 54) | (w5 << 10)) & mask; - out[6] = ((w5 >> 52) | (w6 << 12)) & mask; - out[7] = ((w6 >> 50) | (w7 << 14)) & mask; - out[8] = ((w7 >> 48) | (w8 << 16)) & mask; - out[9] = ((w8 >> 46) | (w9 << 18)) & mask; - out[10] = ((w9 >> 44) | (w10 << 20)) & mask; - out[11] = ((w10 >> 42) | (w11 << 22)) & mask; - out[12] = ((w11 >> 40) | (w12 << 24)) & mask; - out[13] = ((w12 >> 38) | (w13 << 26)) & mask; - out[14] = ((w13 >> 36) | (w14 << 28)) & mask; - out[15] = ((w14 >> 34) | (w15 << 30)) & mask; - out[16] = ((w15 >> 32) | (w16 << 32)) & mask; - out[17] = ((w16 >> 30) | (w17 << 34)) & mask; - out[18] = ((w17 >> 28) | (w18 << 36)) & mask; - out[19] = ((w18 >> 26) | (w19 << 38)) & mask; - out[20] = ((w19 >> 24) | (w20 << 40)) & mask; - out[21] = ((w20 >> 22) | (w21 << 42)) & mask; - out[22] = ((w21 >> 20) | (w22 << 44)) & mask; - out[23] = ((w22 >> 18) | (w23 << 46)) & mask; - out[24] = ((w23 >> 16) | (w24 << 48)) & mask; - out[25] = ((w24 >> 14) | (w25 << 50)) & mask; - out[26] = ((w25 >> 12) | (w26 << 52)) & mask; - out[27] = ((w26 >> 10) | (w27 << 54)) & mask; - out[28] = ((w27 >> 8) | (w28 << 56)) & mask; - out[29] = ((w28 >> 6) | (w29 << 58)) & mask; - out[30] = ((w29 >> 4) | (w30 << 60)) & mask; - out[31] = w30 >> 2; - - return in; -} - -inline const uint8_t* unpack63_64(const uint8_t* in, uint64_t* out) { - const uint64_t mask = 9223372036854775807ULL; - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - uint64_t w21 = util::SafeLoadAs(in); - w21 = arrow::bit_util::FromLittleEndian(w21); - in += 8; - uint64_t w22 = util::SafeLoadAs(in); - w22 = arrow::bit_util::FromLittleEndian(w22); - in += 8; - uint64_t w23 = util::SafeLoadAs(in); - w23 = arrow::bit_util::FromLittleEndian(w23); - in += 8; - uint64_t w24 = util::SafeLoadAs(in); - w24 = arrow::bit_util::FromLittleEndian(w24); - in += 8; - uint64_t w25 = util::SafeLoadAs(in); - w25 = arrow::bit_util::FromLittleEndian(w25); - in += 8; - uint64_t w26 = util::SafeLoadAs(in); - w26 = arrow::bit_util::FromLittleEndian(w26); - in += 8; - uint64_t w27 = util::SafeLoadAs(in); - w27 = arrow::bit_util::FromLittleEndian(w27); - in += 8; - uint64_t w28 = util::SafeLoadAs(in); - w28 = arrow::bit_util::FromLittleEndian(w28); - in += 8; - uint64_t w29 = util::SafeLoadAs(in); - w29 = arrow::bit_util::FromLittleEndian(w29); - in += 8; - uint64_t w30 = util::SafeLoadAs(in); - w30 = arrow::bit_util::FromLittleEndian(w30); - in += 8; - uint64_t w31 = util::SafeLoadAs(in); - w31 = arrow::bit_util::FromLittleEndian(w31); - in += 4; - out[0] = (w0)&mask; - out[1] = ((w0 >> 63) | (w1 << 1)) & mask; - out[2] = ((w1 >> 62) | (w2 << 2)) & mask; - out[3] = ((w2 >> 61) | (w3 << 3)) & mask; - out[4] = ((w3 >> 60) | (w4 << 4)) & mask; - out[5] = ((w4 >> 59) | (w5 << 5)) & mask; - out[6] = ((w5 >> 58) | (w6 << 6)) & mask; - out[7] = ((w6 >> 57) | (w7 << 7)) & mask; - out[8] = ((w7 >> 56) | (w8 << 8)) & mask; - out[9] = ((w8 >> 55) | (w9 << 9)) & mask; - out[10] = ((w9 >> 54) | (w10 << 10)) & mask; - out[11] = ((w10 >> 53) | (w11 << 11)) & mask; - out[12] = ((w11 >> 52) | (w12 << 12)) & mask; - out[13] = ((w12 >> 51) | (w13 << 13)) & mask; - out[14] = ((w13 >> 50) | (w14 << 14)) & mask; - out[15] = ((w14 >> 49) | (w15 << 15)) & mask; - out[16] = ((w15 >> 48) | (w16 << 16)) & mask; - out[17] = ((w16 >> 47) | (w17 << 17)) & mask; - out[18] = ((w17 >> 46) | (w18 << 18)) & mask; - out[19] = ((w18 >> 45) | (w19 << 19)) & mask; - out[20] = ((w19 >> 44) | (w20 << 20)) & mask; - out[21] = ((w20 >> 43) | (w21 << 21)) & mask; - out[22] = ((w21 >> 42) | (w22 << 22)) & mask; - out[23] = ((w22 >> 41) | (w23 << 23)) & mask; - out[24] = ((w23 >> 40) | (w24 << 24)) & mask; - out[25] = ((w24 >> 39) | (w25 << 25)) & mask; - out[26] = ((w25 >> 38) | (w26 << 26)) & mask; - out[27] = ((w26 >> 37) | (w27 << 27)) & mask; - out[28] = ((w27 >> 36) | (w28 << 28)) & mask; - out[29] = ((w28 >> 35) | (w29 << 29)) & mask; - out[30] = ((w29 >> 34) | (w30 << 30)) & mask; - out[31] = ((w30 >> 33) | (w31 << 31)) & mask; - - return in; -} - -inline const uint8_t* unpack64_64(const uint8_t* in, uint64_t* out) { - uint64_t w0 = util::SafeLoadAs(in); - w0 = arrow::bit_util::FromLittleEndian(w0); - in += 8; - uint64_t w1 = util::SafeLoadAs(in); - w1 = arrow::bit_util::FromLittleEndian(w1); - in += 8; - uint64_t w2 = util::SafeLoadAs(in); - w2 = arrow::bit_util::FromLittleEndian(w2); - in += 8; - uint64_t w3 = util::SafeLoadAs(in); - w3 = arrow::bit_util::FromLittleEndian(w3); - in += 8; - uint64_t w4 = util::SafeLoadAs(in); - w4 = arrow::bit_util::FromLittleEndian(w4); - in += 8; - uint64_t w5 = util::SafeLoadAs(in); - w5 = arrow::bit_util::FromLittleEndian(w5); - in += 8; - uint64_t w6 = util::SafeLoadAs(in); - w6 = arrow::bit_util::FromLittleEndian(w6); - in += 8; - uint64_t w7 = util::SafeLoadAs(in); - w7 = arrow::bit_util::FromLittleEndian(w7); - in += 8; - uint64_t w8 = util::SafeLoadAs(in); - w8 = arrow::bit_util::FromLittleEndian(w8); - in += 8; - uint64_t w9 = util::SafeLoadAs(in); - w9 = arrow::bit_util::FromLittleEndian(w9); - in += 8; - uint64_t w10 = util::SafeLoadAs(in); - w10 = arrow::bit_util::FromLittleEndian(w10); - in += 8; - uint64_t w11 = util::SafeLoadAs(in); - w11 = arrow::bit_util::FromLittleEndian(w11); - in += 8; - uint64_t w12 = util::SafeLoadAs(in); - w12 = arrow::bit_util::FromLittleEndian(w12); - in += 8; - uint64_t w13 = util::SafeLoadAs(in); - w13 = arrow::bit_util::FromLittleEndian(w13); - in += 8; - uint64_t w14 = util::SafeLoadAs(in); - w14 = arrow::bit_util::FromLittleEndian(w14); - in += 8; - uint64_t w15 = util::SafeLoadAs(in); - w15 = arrow::bit_util::FromLittleEndian(w15); - in += 8; - uint64_t w16 = util::SafeLoadAs(in); - w16 = arrow::bit_util::FromLittleEndian(w16); - in += 8; - uint64_t w17 = util::SafeLoadAs(in); - w17 = arrow::bit_util::FromLittleEndian(w17); - in += 8; - uint64_t w18 = util::SafeLoadAs(in); - w18 = arrow::bit_util::FromLittleEndian(w18); - in += 8; - uint64_t w19 = util::SafeLoadAs(in); - w19 = arrow::bit_util::FromLittleEndian(w19); - in += 8; - uint64_t w20 = util::SafeLoadAs(in); - w20 = arrow::bit_util::FromLittleEndian(w20); - in += 8; - uint64_t w21 = util::SafeLoadAs(in); - w21 = arrow::bit_util::FromLittleEndian(w21); - in += 8; - uint64_t w22 = util::SafeLoadAs(in); - w22 = arrow::bit_util::FromLittleEndian(w22); - in += 8; - uint64_t w23 = util::SafeLoadAs(in); - w23 = arrow::bit_util::FromLittleEndian(w23); - in += 8; - uint64_t w24 = util::SafeLoadAs(in); - w24 = arrow::bit_util::FromLittleEndian(w24); - in += 8; - uint64_t w25 = util::SafeLoadAs(in); - w25 = arrow::bit_util::FromLittleEndian(w25); - in += 8; - uint64_t w26 = util::SafeLoadAs(in); - w26 = arrow::bit_util::FromLittleEndian(w26); - in += 8; - uint64_t w27 = util::SafeLoadAs(in); - w27 = arrow::bit_util::FromLittleEndian(w27); - in += 8; - uint64_t w28 = util::SafeLoadAs(in); - w28 = arrow::bit_util::FromLittleEndian(w28); - in += 8; - uint64_t w29 = util::SafeLoadAs(in); - w29 = arrow::bit_util::FromLittleEndian(w29); - in += 8; - uint64_t w30 = util::SafeLoadAs(in); - w30 = arrow::bit_util::FromLittleEndian(w30); - in += 8; - uint64_t w31 = util::SafeLoadAs(in); - w31 = arrow::bit_util::FromLittleEndian(w31); - in += 8; - out[0] = w0; - out[1] = w1; - out[2] = w2; - out[3] = w3; - out[4] = w4; - out[5] = w5; - out[6] = w6; - out[7] = w7; - out[8] = w8; - out[9] = w9; - out[10] = w10; - out[11] = w11; - out[12] = w12; - out[13] = w13; - out[14] = w14; - out[15] = w15; - out[16] = w16; - out[17] = w17; - out[18] = w18; - out[19] = w19; - out[20] = w20; - out[21] = w21; - out[22] = w22; - out[23] = w23; - out[24] = w24; - out[25] = w25; - out[26] = w26; - out[27] = w27; - out[28] = w28; - out[29] = w29; - out[30] = w30; - out[31] = w31; - - return in; -} - -} // namespace arrow::internal diff --git a/cpp/src/arrow/util/bpacking_avx2_internal.h b/cpp/src/arrow/util/bpacking_avx2_internal.h deleted file mode 100644 index b2c213fe2aac4..0000000000000 --- a/cpp/src/arrow/util/bpacking_avx2_internal.h +++ /dev/null @@ -1,29 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "arrow/util/visibility.h" - -#include - -namespace arrow::internal { - -ARROW_EXPORT int unpack32_avx2(const uint8_t* in, uint32_t* out, int batch_size, - int num_bits); - -} // namespace arrow::internal diff --git a/cpp/src/arrow/util/bpacking_avx512_internal.h b/cpp/src/arrow/util/bpacking_avx512_internal.h deleted file mode 100644 index 847aa9814336f..0000000000000 --- a/cpp/src/arrow/util/bpacking_avx512_internal.h +++ /dev/null @@ -1,29 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "arrow/util/visibility.h" - -#include - -namespace arrow::internal { - -ARROW_EXPORT int unpack32_avx512(const uint8_t* in, uint32_t* out, int batch_size, - int num_bits); - -} // namespace arrow::internal diff --git a/cpp/src/arrow/util/bpacking_benchmark.cc b/cpp/src/arrow/util/bpacking_benchmark.cc index f0ac22910c657..f281fdf17d369 100644 --- a/cpp/src/arrow/util/bpacking_benchmark.cc +++ b/cpp/src/arrow/util/bpacking_benchmark.cc @@ -22,17 +22,12 @@ #include "arrow/testing/util.h" #include "arrow/util/bpacking_internal.h" +#include "arrow/util/bpacking_scalar_internal.h" +#include "arrow/util/bpacking_simd_internal.h" #if defined(ARROW_HAVE_RUNTIME_AVX2) -# include "arrow/util/bpacking_avx2_internal.h" # include "arrow/util/cpu_info.h" #endif -#if defined(ARROW_HAVE_RUNTIME_AVX512) -# include "arrow/util/bpacking_avx512_internal.h" -#endif -#if defined(ARROW_HAVE_NEON) -# include "arrow/util/bpacking_neon_internal.h" -#endif namespace arrow::internal { namespace { @@ -102,8 +97,13 @@ void BM_Unpack(benchmark::State& state, bool aligned, UnpackFunc unpack, bo constexpr int32_t kMinRange = 64; constexpr int32_t kMaxRange = 32768; +constexpr std::initializer_list kBitWidths16 = {1, 2, 8, 13}; constexpr std::initializer_list kBitWidths32 = {1, 2, 8, 20}; constexpr std::initializer_list kBitWidths64 = {1, 2, 8, 20, 47}; +static const std::vector> kBitWidthsNumValues16 = { + kBitWidths16, + benchmark::CreateRange(kMinRange, kMaxRange, /*multi=*/32), +}; static const std::vector> kBitWidthsNumValues32 = { kBitWidths32, benchmark::CreateRange(kMinRange, kMaxRange, /*multi=*/32), @@ -113,6 +113,11 @@ static const std::vector> kBitWidthsNumValues64 = { benchmark::CreateRange(kMinRange, kMaxRange, /*multi=*/32), }; +/// Nudge for MSVC template inside BENCHMARK_CAPTURE macro. +void BM_UnpackUint16(benchmark::State& state, bool aligned, UnpackFunc unpack, + bool skip = false, std::string skip_msg = "") { + return BM_Unpack(state, aligned, unpack, skip, std::move(skip_msg)); +} /// Nudge for MSVC template inside BENCHMARK_CAPTURE macro. void BM_UnpackUint32(benchmark::State& state, bool aligned, UnpackFunc unpack, bool skip = false, std::string skip_msg = "") { @@ -124,38 +129,70 @@ void BM_UnpackUint64(benchmark::State& state, bool aligned, UnpackFunc return BM_Unpack(state, aligned, unpack, skip, std::move(skip_msg)); } -BENCHMARK_CAPTURE(BM_UnpackUint32, ScalarUnaligned, false, unpack32_scalar) +BENCHMARK_CAPTURE(BM_UnpackUint16, ScalarUnaligned, false, &unpack_scalar) + ->ArgsProduct(kBitWidthsNumValues16); +BENCHMARK_CAPTURE(BM_UnpackUint32, ScalarUnaligned, false, &unpack_scalar) ->ArgsProduct(kBitWidthsNumValues32); -BENCHMARK_CAPTURE(BM_UnpackUint64, ScalarUnaligned, false, unpack64_scalar) +BENCHMARK_CAPTURE(BM_UnpackUint64, ScalarUnaligned, false, &unpack_scalar) ->ArgsProduct(kBitWidthsNumValues64); +#if defined(ARROW_HAVE_SSE4_2) +BENCHMARK_CAPTURE(BM_UnpackUint16, Sse42Unaligned, false, &unpack_sse4_2) + ->ArgsProduct(kBitWidthsNumValues16); +BENCHMARK_CAPTURE(BM_UnpackUint32, Sse42Unaligned, false, &unpack_sse4_2) + ->ArgsProduct(kBitWidthsNumValues32); +BENCHMARK_CAPTURE(BM_UnpackUint64, Sse42Unaligned, false, &unpack_sse4_2) + ->ArgsProduct(kBitWidthsNumValues64); +#endif + #if defined(ARROW_HAVE_RUNTIME_AVX2) -BENCHMARK_CAPTURE(BM_UnpackUint32, Avx2Unaligned, false, unpack32_avx2, +BENCHMARK_CAPTURE(BM_UnpackUint16, Avx2Unaligned, false, &unpack_avx2, + !CpuInfo::GetInstance()->IsSupported(CpuInfo::AVX2), + "Avx2 not available") + ->ArgsProduct(kBitWidthsNumValues16); +BENCHMARK_CAPTURE(BM_UnpackUint32, Avx2Unaligned, false, &unpack_avx2, !CpuInfo::GetInstance()->IsSupported(CpuInfo::AVX2), "Avx2 not available") ->ArgsProduct(kBitWidthsNumValues32); +BENCHMARK_CAPTURE(BM_UnpackUint64, Avx2Unaligned, false, &unpack_avx2, + !CpuInfo::GetInstance()->IsSupported(CpuInfo::AVX2), + "Avx2 not available") + ->ArgsProduct(kBitWidthsNumValues64); #endif #if defined(ARROW_HAVE_RUNTIME_AVX512) -BENCHMARK_CAPTURE(BM_UnpackUint32, Avx512Unaligned, false, unpack32_avx512, +BENCHMARK_CAPTURE(BM_UnpackUint32, Avx512Unaligned, false, &unpack_avx512, !CpuInfo::GetInstance()->IsSupported(CpuInfo::AVX512), "Avx512 not available") ->ArgsProduct(kBitWidthsNumValues32); +BENCHMARK_CAPTURE(BM_UnpackUint64, Avx512Unaligned, false, &unpack_avx512, + !CpuInfo::GetInstance()->IsSupported(CpuInfo::AVX512), + "Avx512 not available") + ->ArgsProduct(kBitWidthsNumValues64); #endif #if defined(ARROW_HAVE_NEON) -BENCHMARK_CAPTURE(BM_UnpackUint32, NeonUnaligned, false, unpack32_neon) +BENCHMARK_CAPTURE(BM_UnpackUint16, NeonUnaligned, false, &unpack_neon) + ->ArgsProduct(kBitWidthsNumValues16); +BENCHMARK_CAPTURE(BM_UnpackUint32, NeonUnaligned, false, &unpack_neon) ->ArgsProduct(kBitWidthsNumValues32); +BENCHMARK_CAPTURE(BM_UnpackUint64, NeonUnaligned, false, &unpack_neon) + ->ArgsProduct(kBitWidthsNumValues64); #endif -BENCHMARK_CAPTURE(BM_UnpackUint32, DynamicAligned, true, unpack32) +BENCHMARK_CAPTURE(BM_UnpackUint16, DynamicAligned, true, &unpack) + ->ArgsProduct(kBitWidthsNumValues16); +BENCHMARK_CAPTURE(BM_UnpackUint16, DynamicUnaligned, false, &unpack) + ->ArgsProduct(kBitWidthsNumValues16); + +BENCHMARK_CAPTURE(BM_UnpackUint32, DynamicAligned, true, &unpack) ->ArgsProduct(kBitWidthsNumValues32); -BENCHMARK_CAPTURE(BM_UnpackUint32, DynamicUnaligned, false, unpack32) +BENCHMARK_CAPTURE(BM_UnpackUint32, DynamicUnaligned, false, &unpack) ->ArgsProduct(kBitWidthsNumValues32); -BENCHMARK_CAPTURE(BM_UnpackUint64, DynamicAligned, true, unpack64) +BENCHMARK_CAPTURE(BM_UnpackUint64, DynamicAligned, true, &unpack) ->ArgsProduct(kBitWidthsNumValues64); -BENCHMARK_CAPTURE(BM_UnpackUint64, DynamicUnaligned, false, unpack64) +BENCHMARK_CAPTURE(BM_UnpackUint64, DynamicUnaligned, false, &unpack) ->ArgsProduct(kBitWidthsNumValues64); } // namespace diff --git a/cpp/src/arrow/util/bpacking_default_internal.h b/cpp/src/arrow/util/bpacking_default_internal.h deleted file mode 100644 index 4c661dcce3798..0000000000000 --- a/cpp/src/arrow/util/bpacking_default_internal.h +++ /dev/null @@ -1,4251 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// This file was modified from its original version for inclusion in parquet-cpp. -// Original source: -// https://github.com/lemire/FrameOfReference/blob/6ccaf9e97160f9a3b299e23a8ef739e711ef0c71/src/bpacking.cpp -// The original copyright notice follows. - -// This code is released under the -// Apache License Version 2.0 http://www.apache.org/licenses/. -// (c) Daniel Lemire 2013 - -#pragma once - -#include "arrow/util/bit_util.h" -#include "arrow/util/ubsan.h" - -namespace arrow { -namespace internal { - -inline const uint32_t* unpack1_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) & 1; - out++; - *out = (inl >> 1) & 1; - out++; - *out = (inl >> 2) & 1; - out++; - *out = (inl >> 3) & 1; - out++; - *out = (inl >> 4) & 1; - out++; - *out = (inl >> 5) & 1; - out++; - *out = (inl >> 6) & 1; - out++; - *out = (inl >> 7) & 1; - out++; - *out = (inl >> 8) & 1; - out++; - *out = (inl >> 9) & 1; - out++; - *out = (inl >> 10) & 1; - out++; - *out = (inl >> 11) & 1; - out++; - *out = (inl >> 12) & 1; - out++; - *out = (inl >> 13) & 1; - out++; - *out = (inl >> 14) & 1; - out++; - *out = (inl >> 15) & 1; - out++; - *out = (inl >> 16) & 1; - out++; - *out = (inl >> 17) & 1; - out++; - *out = (inl >> 18) & 1; - out++; - *out = (inl >> 19) & 1; - out++; - *out = (inl >> 20) & 1; - out++; - *out = (inl >> 21) & 1; - out++; - *out = (inl >> 22) & 1; - out++; - *out = (inl >> 23) & 1; - out++; - *out = (inl >> 24) & 1; - out++; - *out = (inl >> 25) & 1; - out++; - *out = (inl >> 26) & 1; - out++; - *out = (inl >> 27) & 1; - out++; - *out = (inl >> 28) & 1; - out++; - *out = (inl >> 29) & 1; - out++; - *out = (inl >> 30) & 1; - out++; - *out = (inl >> 31); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack2_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 2); - out++; - *out = (inl >> 2) % (1U << 2); - out++; - *out = (inl >> 4) % (1U << 2); - out++; - *out = (inl >> 6) % (1U << 2); - out++; - *out = (inl >> 8) % (1U << 2); - out++; - *out = (inl >> 10) % (1U << 2); - out++; - *out = (inl >> 12) % (1U << 2); - out++; - *out = (inl >> 14) % (1U << 2); - out++; - *out = (inl >> 16) % (1U << 2); - out++; - *out = (inl >> 18) % (1U << 2); - out++; - *out = (inl >> 20) % (1U << 2); - out++; - *out = (inl >> 22) % (1U << 2); - out++; - *out = (inl >> 24) % (1U << 2); - out++; - *out = (inl >> 26) % (1U << 2); - out++; - *out = (inl >> 28) % (1U << 2); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 2); - out++; - *out = (inl >> 2) % (1U << 2); - out++; - *out = (inl >> 4) % (1U << 2); - out++; - *out = (inl >> 6) % (1U << 2); - out++; - *out = (inl >> 8) % (1U << 2); - out++; - *out = (inl >> 10) % (1U << 2); - out++; - *out = (inl >> 12) % (1U << 2); - out++; - *out = (inl >> 14) % (1U << 2); - out++; - *out = (inl >> 16) % (1U << 2); - out++; - *out = (inl >> 18) % (1U << 2); - out++; - *out = (inl >> 20) % (1U << 2); - out++; - *out = (inl >> 22) % (1U << 2); - out++; - *out = (inl >> 24) % (1U << 2); - out++; - *out = (inl >> 26) % (1U << 2); - out++; - *out = (inl >> 28) % (1U << 2); - out++; - *out = (inl >> 30); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack3_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 3); - out++; - *out = (inl >> 3) % (1U << 3); - out++; - *out = (inl >> 6) % (1U << 3); - out++; - *out = (inl >> 9) % (1U << 3); - out++; - *out = (inl >> 12) % (1U << 3); - out++; - *out = (inl >> 15) % (1U << 3); - out++; - *out = (inl >> 18) % (1U << 3); - out++; - *out = (inl >> 21) % (1U << 3); - out++; - *out = (inl >> 24) % (1U << 3); - out++; - *out = (inl >> 27) % (1U << 3); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 1)) << (3 - 1); - out++; - *out = (inl >> 1) % (1U << 3); - out++; - *out = (inl >> 4) % (1U << 3); - out++; - *out = (inl >> 7) % (1U << 3); - out++; - *out = (inl >> 10) % (1U << 3); - out++; - *out = (inl >> 13) % (1U << 3); - out++; - *out = (inl >> 16) % (1U << 3); - out++; - *out = (inl >> 19) % (1U << 3); - out++; - *out = (inl >> 22) % (1U << 3); - out++; - *out = (inl >> 25) % (1U << 3); - out++; - *out = (inl >> 28) % (1U << 3); - out++; - *out = (inl >> 31); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (3 - 2); - out++; - *out = (inl >> 2) % (1U << 3); - out++; - *out = (inl >> 5) % (1U << 3); - out++; - *out = (inl >> 8) % (1U << 3); - out++; - *out = (inl >> 11) % (1U << 3); - out++; - *out = (inl >> 14) % (1U << 3); - out++; - *out = (inl >> 17) % (1U << 3); - out++; - *out = (inl >> 20) % (1U << 3); - out++; - *out = (inl >> 23) % (1U << 3); - out++; - *out = (inl >> 26) % (1U << 3); - out++; - *out = (inl >> 29); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack4_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 4); - out++; - *out = (inl >> 4) % (1U << 4); - out++; - *out = (inl >> 8) % (1U << 4); - out++; - *out = (inl >> 12) % (1U << 4); - out++; - *out = (inl >> 16) % (1U << 4); - out++; - *out = (inl >> 20) % (1U << 4); - out++; - *out = (inl >> 24) % (1U << 4); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 4); - out++; - *out = (inl >> 4) % (1U << 4); - out++; - *out = (inl >> 8) % (1U << 4); - out++; - *out = (inl >> 12) % (1U << 4); - out++; - *out = (inl >> 16) % (1U << 4); - out++; - *out = (inl >> 20) % (1U << 4); - out++; - *out = (inl >> 24) % (1U << 4); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 4); - out++; - *out = (inl >> 4) % (1U << 4); - out++; - *out = (inl >> 8) % (1U << 4); - out++; - *out = (inl >> 12) % (1U << 4); - out++; - *out = (inl >> 16) % (1U << 4); - out++; - *out = (inl >> 20) % (1U << 4); - out++; - *out = (inl >> 24) % (1U << 4); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 4); - out++; - *out = (inl >> 4) % (1U << 4); - out++; - *out = (inl >> 8) % (1U << 4); - out++; - *out = (inl >> 12) % (1U << 4); - out++; - *out = (inl >> 16) % (1U << 4); - out++; - *out = (inl >> 20) % (1U << 4); - out++; - *out = (inl >> 24) % (1U << 4); - out++; - *out = (inl >> 28); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack5_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 5); - out++; - *out = (inl >> 5) % (1U << 5); - out++; - *out = (inl >> 10) % (1U << 5); - out++; - *out = (inl >> 15) % (1U << 5); - out++; - *out = (inl >> 20) % (1U << 5); - out++; - *out = (inl >> 25) % (1U << 5); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 3)) << (5 - 3); - out++; - *out = (inl >> 3) % (1U << 5); - out++; - *out = (inl >> 8) % (1U << 5); - out++; - *out = (inl >> 13) % (1U << 5); - out++; - *out = (inl >> 18) % (1U << 5); - out++; - *out = (inl >> 23) % (1U << 5); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 1)) << (5 - 1); - out++; - *out = (inl >> 1) % (1U << 5); - out++; - *out = (inl >> 6) % (1U << 5); - out++; - *out = (inl >> 11) % (1U << 5); - out++; - *out = (inl >> 16) % (1U << 5); - out++; - *out = (inl >> 21) % (1U << 5); - out++; - *out = (inl >> 26) % (1U << 5); - out++; - *out = (inl >> 31); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (5 - 4); - out++; - *out = (inl >> 4) % (1U << 5); - out++; - *out = (inl >> 9) % (1U << 5); - out++; - *out = (inl >> 14) % (1U << 5); - out++; - *out = (inl >> 19) % (1U << 5); - out++; - *out = (inl >> 24) % (1U << 5); - out++; - *out = (inl >> 29); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (5 - 2); - out++; - *out = (inl >> 2) % (1U << 5); - out++; - *out = (inl >> 7) % (1U << 5); - out++; - *out = (inl >> 12) % (1U << 5); - out++; - *out = (inl >> 17) % (1U << 5); - out++; - *out = (inl >> 22) % (1U << 5); - out++; - *out = (inl >> 27); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack6_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 6); - out++; - *out = (inl >> 6) % (1U << 6); - out++; - *out = (inl >> 12) % (1U << 6); - out++; - *out = (inl >> 18) % (1U << 6); - out++; - *out = (inl >> 24) % (1U << 6); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (6 - 4); - out++; - *out = (inl >> 4) % (1U << 6); - out++; - *out = (inl >> 10) % (1U << 6); - out++; - *out = (inl >> 16) % (1U << 6); - out++; - *out = (inl >> 22) % (1U << 6); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (6 - 2); - out++; - *out = (inl >> 2) % (1U << 6); - out++; - *out = (inl >> 8) % (1U << 6); - out++; - *out = (inl >> 14) % (1U << 6); - out++; - *out = (inl >> 20) % (1U << 6); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 6); - out++; - *out = (inl >> 6) % (1U << 6); - out++; - *out = (inl >> 12) % (1U << 6); - out++; - *out = (inl >> 18) % (1U << 6); - out++; - *out = (inl >> 24) % (1U << 6); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (6 - 4); - out++; - *out = (inl >> 4) % (1U << 6); - out++; - *out = (inl >> 10) % (1U << 6); - out++; - *out = (inl >> 16) % (1U << 6); - out++; - *out = (inl >> 22) % (1U << 6); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (6 - 2); - out++; - *out = (inl >> 2) % (1U << 6); - out++; - *out = (inl >> 8) % (1U << 6); - out++; - *out = (inl >> 14) % (1U << 6); - out++; - *out = (inl >> 20) % (1U << 6); - out++; - *out = (inl >> 26); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack7_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 7); - out++; - *out = (inl >> 7) % (1U << 7); - out++; - *out = (inl >> 14) % (1U << 7); - out++; - *out = (inl >> 21) % (1U << 7); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 3)) << (7 - 3); - out++; - *out = (inl >> 3) % (1U << 7); - out++; - *out = (inl >> 10) % (1U << 7); - out++; - *out = (inl >> 17) % (1U << 7); - out++; - *out = (inl >> 24) % (1U << 7); - out++; - *out = (inl >> 31); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (7 - 6); - out++; - *out = (inl >> 6) % (1U << 7); - out++; - *out = (inl >> 13) % (1U << 7); - out++; - *out = (inl >> 20) % (1U << 7); - out++; - *out = (inl >> 27); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (7 - 2); - out++; - *out = (inl >> 2) % (1U << 7); - out++; - *out = (inl >> 9) % (1U << 7); - out++; - *out = (inl >> 16) % (1U << 7); - out++; - *out = (inl >> 23) % (1U << 7); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 5)) << (7 - 5); - out++; - *out = (inl >> 5) % (1U << 7); - out++; - *out = (inl >> 12) % (1U << 7); - out++; - *out = (inl >> 19) % (1U << 7); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 1)) << (7 - 1); - out++; - *out = (inl >> 1) % (1U << 7); - out++; - *out = (inl >> 8) % (1U << 7); - out++; - *out = (inl >> 15) % (1U << 7); - out++; - *out = (inl >> 22) % (1U << 7); - out++; - *out = (inl >> 29); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (7 - 4); - out++; - *out = (inl >> 4) % (1U << 7); - out++; - *out = (inl >> 11) % (1U << 7); - out++; - *out = (inl >> 18) % (1U << 7); - out++; - *out = (inl >> 25); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack8_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 8); - out++; - *out = (inl >> 8) % (1U << 8); - out++; - *out = (inl >> 16) % (1U << 8); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 8); - out++; - *out = (inl >> 8) % (1U << 8); - out++; - *out = (inl >> 16) % (1U << 8); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 8); - out++; - *out = (inl >> 8) % (1U << 8); - out++; - *out = (inl >> 16) % (1U << 8); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 8); - out++; - *out = (inl >> 8) % (1U << 8); - out++; - *out = (inl >> 16) % (1U << 8); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 8); - out++; - *out = (inl >> 8) % (1U << 8); - out++; - *out = (inl >> 16) % (1U << 8); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 8); - out++; - *out = (inl >> 8) % (1U << 8); - out++; - *out = (inl >> 16) % (1U << 8); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 8); - out++; - *out = (inl >> 8) % (1U << 8); - out++; - *out = (inl >> 16) % (1U << 8); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 8); - out++; - *out = (inl >> 8) % (1U << 8); - out++; - *out = (inl >> 16) % (1U << 8); - out++; - *out = (inl >> 24); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack9_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 9); - out++; - *out = (inl >> 9) % (1U << 9); - out++; - *out = (inl >> 18) % (1U << 9); - out++; - *out = (inl >> 27); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (9 - 4); - out++; - *out = (inl >> 4) % (1U << 9); - out++; - *out = (inl >> 13) % (1U << 9); - out++; - *out = (inl >> 22) % (1U << 9); - out++; - *out = (inl >> 31); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (9 - 8); - out++; - *out = (inl >> 8) % (1U << 9); - out++; - *out = (inl >> 17) % (1U << 9); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 3)) << (9 - 3); - out++; - *out = (inl >> 3) % (1U << 9); - out++; - *out = (inl >> 12) % (1U << 9); - out++; - *out = (inl >> 21) % (1U << 9); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 7)) << (9 - 7); - out++; - *out = (inl >> 7) % (1U << 9); - out++; - *out = (inl >> 16) % (1U << 9); - out++; - *out = (inl >> 25); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (9 - 2); - out++; - *out = (inl >> 2) % (1U << 9); - out++; - *out = (inl >> 11) % (1U << 9); - out++; - *out = (inl >> 20) % (1U << 9); - out++; - *out = (inl >> 29); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (9 - 6); - out++; - *out = (inl >> 6) % (1U << 9); - out++; - *out = (inl >> 15) % (1U << 9); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 1)) << (9 - 1); - out++; - *out = (inl >> 1) % (1U << 9); - out++; - *out = (inl >> 10) % (1U << 9); - out++; - *out = (inl >> 19) % (1U << 9); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 5)) << (9 - 5); - out++; - *out = (inl >> 5) % (1U << 9); - out++; - *out = (inl >> 14) % (1U << 9); - out++; - *out = (inl >> 23); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack10_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 10); - out++; - *out = (inl >> 10) % (1U << 10); - out++; - *out = (inl >> 20) % (1U << 10); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (10 - 8); - out++; - *out = (inl >> 8) % (1U << 10); - out++; - *out = (inl >> 18) % (1U << 10); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (10 - 6); - out++; - *out = (inl >> 6) % (1U << 10); - out++; - *out = (inl >> 16) % (1U << 10); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (10 - 4); - out++; - *out = (inl >> 4) % (1U << 10); - out++; - *out = (inl >> 14) % (1U << 10); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (10 - 2); - out++; - *out = (inl >> 2) % (1U << 10); - out++; - *out = (inl >> 12) % (1U << 10); - out++; - *out = (inl >> 22); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 10); - out++; - *out = (inl >> 10) % (1U << 10); - out++; - *out = (inl >> 20) % (1U << 10); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (10 - 8); - out++; - *out = (inl >> 8) % (1U << 10); - out++; - *out = (inl >> 18) % (1U << 10); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (10 - 6); - out++; - *out = (inl >> 6) % (1U << 10); - out++; - *out = (inl >> 16) % (1U << 10); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (10 - 4); - out++; - *out = (inl >> 4) % (1U << 10); - out++; - *out = (inl >> 14) % (1U << 10); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (10 - 2); - out++; - *out = (inl >> 2) % (1U << 10); - out++; - *out = (inl >> 12) % (1U << 10); - out++; - *out = (inl >> 22); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack11_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 11); - out++; - *out = (inl >> 11) % (1U << 11); - out++; - *out = (inl >> 22); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 1)) << (11 - 1); - out++; - *out = (inl >> 1) % (1U << 11); - out++; - *out = (inl >> 12) % (1U << 11); - out++; - *out = (inl >> 23); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (11 - 2); - out++; - *out = (inl >> 2) % (1U << 11); - out++; - *out = (inl >> 13) % (1U << 11); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 3)) << (11 - 3); - out++; - *out = (inl >> 3) % (1U << 11); - out++; - *out = (inl >> 14) % (1U << 11); - out++; - *out = (inl >> 25); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (11 - 4); - out++; - *out = (inl >> 4) % (1U << 11); - out++; - *out = (inl >> 15) % (1U << 11); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 5)) << (11 - 5); - out++; - *out = (inl >> 5) % (1U << 11); - out++; - *out = (inl >> 16) % (1U << 11); - out++; - *out = (inl >> 27); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (11 - 6); - out++; - *out = (inl >> 6) % (1U << 11); - out++; - *out = (inl >> 17) % (1U << 11); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 7)) << (11 - 7); - out++; - *out = (inl >> 7) % (1U << 11); - out++; - *out = (inl >> 18) % (1U << 11); - out++; - *out = (inl >> 29); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (11 - 8); - out++; - *out = (inl >> 8) % (1U << 11); - out++; - *out = (inl >> 19) % (1U << 11); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 9)) << (11 - 9); - out++; - *out = (inl >> 9) % (1U << 11); - out++; - *out = (inl >> 20) % (1U << 11); - out++; - *out = (inl >> 31); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 10)) << (11 - 10); - out++; - *out = (inl >> 10) % (1U << 11); - out++; - *out = (inl >> 21); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack12_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 12); - out++; - *out = (inl >> 12) % (1U << 12); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (12 - 4); - out++; - *out = (inl >> 4) % (1U << 12); - out++; - *out = (inl >> 16) % (1U << 12); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (12 - 8); - out++; - *out = (inl >> 8) % (1U << 12); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 12); - out++; - *out = (inl >> 12) % (1U << 12); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (12 - 4); - out++; - *out = (inl >> 4) % (1U << 12); - out++; - *out = (inl >> 16) % (1U << 12); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (12 - 8); - out++; - *out = (inl >> 8) % (1U << 12); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 12); - out++; - *out = (inl >> 12) % (1U << 12); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (12 - 4); - out++; - *out = (inl >> 4) % (1U << 12); - out++; - *out = (inl >> 16) % (1U << 12); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (12 - 8); - out++; - *out = (inl >> 8) % (1U << 12); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 12); - out++; - *out = (inl >> 12) % (1U << 12); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (12 - 4); - out++; - *out = (inl >> 4) % (1U << 12); - out++; - *out = (inl >> 16) % (1U << 12); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (12 - 8); - out++; - *out = (inl >> 8) % (1U << 12); - out++; - *out = (inl >> 20); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack13_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 13); - out++; - *out = (inl >> 13) % (1U << 13); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 7)) << (13 - 7); - out++; - *out = (inl >> 7) % (1U << 13); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 1)) << (13 - 1); - out++; - *out = (inl >> 1) % (1U << 13); - out++; - *out = (inl >> 14) % (1U << 13); - out++; - *out = (inl >> 27); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (13 - 8); - out++; - *out = (inl >> 8) % (1U << 13); - out++; - *out = (inl >> 21); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (13 - 2); - out++; - *out = (inl >> 2) % (1U << 13); - out++; - *out = (inl >> 15) % (1U << 13); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 9)) << (13 - 9); - out++; - *out = (inl >> 9) % (1U << 13); - out++; - *out = (inl >> 22); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 3)) << (13 - 3); - out++; - *out = (inl >> 3) % (1U << 13); - out++; - *out = (inl >> 16) % (1U << 13); - out++; - *out = (inl >> 29); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 10)) << (13 - 10); - out++; - *out = (inl >> 10) % (1U << 13); - out++; - *out = (inl >> 23); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (13 - 4); - out++; - *out = (inl >> 4) % (1U << 13); - out++; - *out = (inl >> 17) % (1U << 13); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 11)) << (13 - 11); - out++; - *out = (inl >> 11) % (1U << 13); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 5)) << (13 - 5); - out++; - *out = (inl >> 5) % (1U << 13); - out++; - *out = (inl >> 18) % (1U << 13); - out++; - *out = (inl >> 31); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (13 - 12); - out++; - *out = (inl >> 12) % (1U << 13); - out++; - *out = (inl >> 25); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (13 - 6); - out++; - *out = (inl >> 6) % (1U << 13); - out++; - *out = (inl >> 19); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack14_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 14); - out++; - *out = (inl >> 14) % (1U << 14); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 10)) << (14 - 10); - out++; - *out = (inl >> 10) % (1U << 14); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (14 - 6); - out++; - *out = (inl >> 6) % (1U << 14); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (14 - 2); - out++; - *out = (inl >> 2) % (1U << 14); - out++; - *out = (inl >> 16) % (1U << 14); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (14 - 12); - out++; - *out = (inl >> 12) % (1U << 14); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (14 - 8); - out++; - *out = (inl >> 8) % (1U << 14); - out++; - *out = (inl >> 22); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (14 - 4); - out++; - *out = (inl >> 4) % (1U << 14); - out++; - *out = (inl >> 18); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 14); - out++; - *out = (inl >> 14) % (1U << 14); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 10)) << (14 - 10); - out++; - *out = (inl >> 10) % (1U << 14); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (14 - 6); - out++; - *out = (inl >> 6) % (1U << 14); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (14 - 2); - out++; - *out = (inl >> 2) % (1U << 14); - out++; - *out = (inl >> 16) % (1U << 14); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (14 - 12); - out++; - *out = (inl >> 12) % (1U << 14); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (14 - 8); - out++; - *out = (inl >> 8) % (1U << 14); - out++; - *out = (inl >> 22); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (14 - 4); - out++; - *out = (inl >> 4) % (1U << 14); - out++; - *out = (inl >> 18); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack15_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 15); - out++; - *out = (inl >> 15) % (1U << 15); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 13)) << (15 - 13); - out++; - *out = (inl >> 13) % (1U << 15); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 11)) << (15 - 11); - out++; - *out = (inl >> 11) % (1U << 15); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 9)) << (15 - 9); - out++; - *out = (inl >> 9) % (1U << 15); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 7)) << (15 - 7); - out++; - *out = (inl >> 7) % (1U << 15); - out++; - *out = (inl >> 22); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 5)) << (15 - 5); - out++; - *out = (inl >> 5) % (1U << 15); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 3)) << (15 - 3); - out++; - *out = (inl >> 3) % (1U << 15); - out++; - *out = (inl >> 18); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 1)) << (15 - 1); - out++; - *out = (inl >> 1) % (1U << 15); - out++; - *out = (inl >> 16) % (1U << 15); - out++; - *out = (inl >> 31); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 14)) << (15 - 14); - out++; - *out = (inl >> 14) % (1U << 15); - out++; - *out = (inl >> 29); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (15 - 12); - out++; - *out = (inl >> 12) % (1U << 15); - out++; - *out = (inl >> 27); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 10)) << (15 - 10); - out++; - *out = (inl >> 10) % (1U << 15); - out++; - *out = (inl >> 25); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (15 - 8); - out++; - *out = (inl >> 8) % (1U << 15); - out++; - *out = (inl >> 23); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (15 - 6); - out++; - *out = (inl >> 6) % (1U << 15); - out++; - *out = (inl >> 21); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (15 - 4); - out++; - *out = (inl >> 4) % (1U << 15); - out++; - *out = (inl >> 19); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (15 - 2); - out++; - *out = (inl >> 2) % (1U << 15); - out++; - *out = (inl >> 17); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack16_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 16); - out++; - *out = (inl >> 16); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack17_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 17); - out++; - *out = (inl >> 17); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (17 - 2); - out++; - *out = (inl >> 2) % (1U << 17); - out++; - *out = (inl >> 19); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (17 - 4); - out++; - *out = (inl >> 4) % (1U << 17); - out++; - *out = (inl >> 21); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (17 - 6); - out++; - *out = (inl >> 6) % (1U << 17); - out++; - *out = (inl >> 23); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (17 - 8); - out++; - *out = (inl >> 8) % (1U << 17); - out++; - *out = (inl >> 25); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 10)) << (17 - 10); - out++; - *out = (inl >> 10) % (1U << 17); - out++; - *out = (inl >> 27); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (17 - 12); - out++; - *out = (inl >> 12) % (1U << 17); - out++; - *out = (inl >> 29); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 14)) << (17 - 14); - out++; - *out = (inl >> 14) % (1U << 17); - out++; - *out = (inl >> 31); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (17 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 1)) << (17 - 1); - out++; - *out = (inl >> 1) % (1U << 17); - out++; - *out = (inl >> 18); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 3)) << (17 - 3); - out++; - *out = (inl >> 3) % (1U << 17); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 5)) << (17 - 5); - out++; - *out = (inl >> 5) % (1U << 17); - out++; - *out = (inl >> 22); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 7)) << (17 - 7); - out++; - *out = (inl >> 7) % (1U << 17); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 9)) << (17 - 9); - out++; - *out = (inl >> 9) % (1U << 17); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 11)) << (17 - 11); - out++; - *out = (inl >> 11) % (1U << 17); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 13)) << (17 - 13); - out++; - *out = (inl >> 13) % (1U << 17); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 15)) << (17 - 15); - out++; - *out = (inl >> 15); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack18_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 18); - out++; - *out = (inl >> 18); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (18 - 4); - out++; - *out = (inl >> 4) % (1U << 18); - out++; - *out = (inl >> 22); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (18 - 8); - out++; - *out = (inl >> 8) % (1U << 18); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (18 - 12); - out++; - *out = (inl >> 12) % (1U << 18); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (18 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (18 - 2); - out++; - *out = (inl >> 2) % (1U << 18); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (18 - 6); - out++; - *out = (inl >> 6) % (1U << 18); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 10)) << (18 - 10); - out++; - *out = (inl >> 10) % (1U << 18); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 14)) << (18 - 14); - out++; - *out = (inl >> 14); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 18); - out++; - *out = (inl >> 18); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (18 - 4); - out++; - *out = (inl >> 4) % (1U << 18); - out++; - *out = (inl >> 22); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (18 - 8); - out++; - *out = (inl >> 8) % (1U << 18); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (18 - 12); - out++; - *out = (inl >> 12) % (1U << 18); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (18 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (18 - 2); - out++; - *out = (inl >> 2) % (1U << 18); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (18 - 6); - out++; - *out = (inl >> 6) % (1U << 18); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 10)) << (18 - 10); - out++; - *out = (inl >> 10) % (1U << 18); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 14)) << (18 - 14); - out++; - *out = (inl >> 14); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack19_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 19); - out++; - *out = (inl >> 19); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (19 - 6); - out++; - *out = (inl >> 6) % (1U << 19); - out++; - *out = (inl >> 25); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (19 - 12); - out++; - *out = (inl >> 12) % (1U << 19); - out++; - *out = (inl >> 31); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 18)) << (19 - 18); - out++; - *out = (inl >> 18); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 5)) << (19 - 5); - out++; - *out = (inl >> 5) % (1U << 19); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 11)) << (19 - 11); - out++; - *out = (inl >> 11) % (1U << 19); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 17)) << (19 - 17); - out++; - *out = (inl >> 17); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (19 - 4); - out++; - *out = (inl >> 4) % (1U << 19); - out++; - *out = (inl >> 23); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 10)) << (19 - 10); - out++; - *out = (inl >> 10) % (1U << 19); - out++; - *out = (inl >> 29); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (19 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 3)) << (19 - 3); - out++; - *out = (inl >> 3) % (1U << 19); - out++; - *out = (inl >> 22); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 9)) << (19 - 9); - out++; - *out = (inl >> 9) % (1U << 19); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 15)) << (19 - 15); - out++; - *out = (inl >> 15); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (19 - 2); - out++; - *out = (inl >> 2) % (1U << 19); - out++; - *out = (inl >> 21); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (19 - 8); - out++; - *out = (inl >> 8) % (1U << 19); - out++; - *out = (inl >> 27); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 14)) << (19 - 14); - out++; - *out = (inl >> 14); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 1)) << (19 - 1); - out++; - *out = (inl >> 1) % (1U << 19); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 7)) << (19 - 7); - out++; - *out = (inl >> 7) % (1U << 19); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 13)) << (19 - 13); - out++; - *out = (inl >> 13); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack20_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 20); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (20 - 8); - out++; - *out = (inl >> 8) % (1U << 20); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (20 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (20 - 4); - out++; - *out = (inl >> 4) % (1U << 20); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (20 - 12); - out++; - *out = (inl >> 12); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 20); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (20 - 8); - out++; - *out = (inl >> 8) % (1U << 20); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (20 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (20 - 4); - out++; - *out = (inl >> 4) % (1U << 20); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (20 - 12); - out++; - *out = (inl >> 12); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 20); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (20 - 8); - out++; - *out = (inl >> 8) % (1U << 20); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (20 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (20 - 4); - out++; - *out = (inl >> 4) % (1U << 20); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (20 - 12); - out++; - *out = (inl >> 12); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 20); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (20 - 8); - out++; - *out = (inl >> 8) % (1U << 20); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (20 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (20 - 4); - out++; - *out = (inl >> 4) % (1U << 20); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (20 - 12); - out++; - *out = (inl >> 12); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack21_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 21); - out++; - *out = (inl >> 21); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 10)) << (21 - 10); - out++; - *out = (inl >> 10) % (1U << 21); - out++; - *out = (inl >> 31); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 20)) << (21 - 20); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 9)) << (21 - 9); - out++; - *out = (inl >> 9) % (1U << 21); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 19)) << (21 - 19); - out++; - *out = (inl >> 19); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (21 - 8); - out++; - *out = (inl >> 8) % (1U << 21); - out++; - *out = (inl >> 29); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 18)) << (21 - 18); - out++; - *out = (inl >> 18); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 7)) << (21 - 7); - out++; - *out = (inl >> 7) % (1U << 21); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 17)) << (21 - 17); - out++; - *out = (inl >> 17); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (21 - 6); - out++; - *out = (inl >> 6) % (1U << 21); - out++; - *out = (inl >> 27); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (21 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 5)) << (21 - 5); - out++; - *out = (inl >> 5) % (1U << 21); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 15)) << (21 - 15); - out++; - *out = (inl >> 15); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (21 - 4); - out++; - *out = (inl >> 4) % (1U << 21); - out++; - *out = (inl >> 25); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 14)) << (21 - 14); - out++; - *out = (inl >> 14); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 3)) << (21 - 3); - out++; - *out = (inl >> 3) % (1U << 21); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 13)) << (21 - 13); - out++; - *out = (inl >> 13); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (21 - 2); - out++; - *out = (inl >> 2) % (1U << 21); - out++; - *out = (inl >> 23); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (21 - 12); - out++; - *out = (inl >> 12); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 1)) << (21 - 1); - out++; - *out = (inl >> 1) % (1U << 21); - out++; - *out = (inl >> 22); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 11)) << (21 - 11); - out++; - *out = (inl >> 11); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack22_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 22); - out++; - *out = (inl >> 22); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (22 - 12); - out++; - *out = (inl >> 12); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (22 - 2); - out++; - *out = (inl >> 2) % (1U << 22); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 14)) << (22 - 14); - out++; - *out = (inl >> 14); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (22 - 4); - out++; - *out = (inl >> 4) % (1U << 22); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (22 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (22 - 6); - out++; - *out = (inl >> 6) % (1U << 22); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 18)) << (22 - 18); - out++; - *out = (inl >> 18); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (22 - 8); - out++; - *out = (inl >> 8) % (1U << 22); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 20)) << (22 - 20); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 10)) << (22 - 10); - out++; - *out = (inl >> 10); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 22); - out++; - *out = (inl >> 22); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (22 - 12); - out++; - *out = (inl >> 12); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (22 - 2); - out++; - *out = (inl >> 2) % (1U << 22); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 14)) << (22 - 14); - out++; - *out = (inl >> 14); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (22 - 4); - out++; - *out = (inl >> 4) % (1U << 22); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (22 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (22 - 6); - out++; - *out = (inl >> 6) % (1U << 22); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 18)) << (22 - 18); - out++; - *out = (inl >> 18); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (22 - 8); - out++; - *out = (inl >> 8) % (1U << 22); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 20)) << (22 - 20); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 10)) << (22 - 10); - out++; - *out = (inl >> 10); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack23_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 23); - out++; - *out = (inl >> 23); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 14)) << (23 - 14); - out++; - *out = (inl >> 14); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 5)) << (23 - 5); - out++; - *out = (inl >> 5) % (1U << 23); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 19)) << (23 - 19); - out++; - *out = (inl >> 19); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 10)) << (23 - 10); - out++; - *out = (inl >> 10); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 1)) << (23 - 1); - out++; - *out = (inl >> 1) % (1U << 23); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 15)) << (23 - 15); - out++; - *out = (inl >> 15); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (23 - 6); - out++; - *out = (inl >> 6) % (1U << 23); - out++; - *out = (inl >> 29); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 20)) << (23 - 20); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 11)) << (23 - 11); - out++; - *out = (inl >> 11); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (23 - 2); - out++; - *out = (inl >> 2) % (1U << 23); - out++; - *out = (inl >> 25); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (23 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 7)) << (23 - 7); - out++; - *out = (inl >> 7) % (1U << 23); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 21)) << (23 - 21); - out++; - *out = (inl >> 21); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (23 - 12); - out++; - *out = (inl >> 12); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 3)) << (23 - 3); - out++; - *out = (inl >> 3) % (1U << 23); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 17)) << (23 - 17); - out++; - *out = (inl >> 17); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (23 - 8); - out++; - *out = (inl >> 8) % (1U << 23); - out++; - *out = (inl >> 31); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 22)) << (23 - 22); - out++; - *out = (inl >> 22); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 13)) << (23 - 13); - out++; - *out = (inl >> 13); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (23 - 4); - out++; - *out = (inl >> 4) % (1U << 23); - out++; - *out = (inl >> 27); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 18)) << (23 - 18); - out++; - *out = (inl >> 18); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 9)) << (23 - 9); - out++; - *out = (inl >> 9); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack24_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 24); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (24 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (24 - 8); - out++; - *out = (inl >> 8); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 24); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (24 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (24 - 8); - out++; - *out = (inl >> 8); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 24); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (24 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (24 - 8); - out++; - *out = (inl >> 8); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 24); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (24 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (24 - 8); - out++; - *out = (inl >> 8); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 24); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (24 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (24 - 8); - out++; - *out = (inl >> 8); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 24); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (24 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (24 - 8); - out++; - *out = (inl >> 8); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 24); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (24 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (24 - 8); - out++; - *out = (inl >> 8); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 24); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (24 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (24 - 8); - out++; - *out = (inl >> 8); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack25_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 25); - out++; - *out = (inl >> 25); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 18)) << (25 - 18); - out++; - *out = (inl >> 18); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 11)) << (25 - 11); - out++; - *out = (inl >> 11); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (25 - 4); - out++; - *out = (inl >> 4) % (1U << 25); - out++; - *out = (inl >> 29); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 22)) << (25 - 22); - out++; - *out = (inl >> 22); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 15)) << (25 - 15); - out++; - *out = (inl >> 15); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (25 - 8); - out++; - *out = (inl >> 8); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 1)) << (25 - 1); - out++; - *out = (inl >> 1) % (1U << 25); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 19)) << (25 - 19); - out++; - *out = (inl >> 19); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (25 - 12); - out++; - *out = (inl >> 12); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 5)) << (25 - 5); - out++; - *out = (inl >> 5) % (1U << 25); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 23)) << (25 - 23); - out++; - *out = (inl >> 23); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (25 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 9)) << (25 - 9); - out++; - *out = (inl >> 9); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (25 - 2); - out++; - *out = (inl >> 2) % (1U << 25); - out++; - *out = (inl >> 27); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 20)) << (25 - 20); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 13)) << (25 - 13); - out++; - *out = (inl >> 13); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (25 - 6); - out++; - *out = (inl >> 6) % (1U << 25); - out++; - *out = (inl >> 31); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 24)) << (25 - 24); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 17)) << (25 - 17); - out++; - *out = (inl >> 17); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 10)) << (25 - 10); - out++; - *out = (inl >> 10); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 3)) << (25 - 3); - out++; - *out = (inl >> 3) % (1U << 25); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 21)) << (25 - 21); - out++; - *out = (inl >> 21); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 14)) << (25 - 14); - out++; - *out = (inl >> 14); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 7)) << (25 - 7); - out++; - *out = (inl >> 7); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack26_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 26); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 20)) << (26 - 20); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 14)) << (26 - 14); - out++; - *out = (inl >> 14); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (26 - 8); - out++; - *out = (inl >> 8); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (26 - 2); - out++; - *out = (inl >> 2) % (1U << 26); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 22)) << (26 - 22); - out++; - *out = (inl >> 22); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (26 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 10)) << (26 - 10); - out++; - *out = (inl >> 10); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (26 - 4); - out++; - *out = (inl >> 4) % (1U << 26); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 24)) << (26 - 24); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 18)) << (26 - 18); - out++; - *out = (inl >> 18); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (26 - 12); - out++; - *out = (inl >> 12); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (26 - 6); - out++; - *out = (inl >> 6); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 26); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 20)) << (26 - 20); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 14)) << (26 - 14); - out++; - *out = (inl >> 14); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (26 - 8); - out++; - *out = (inl >> 8); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (26 - 2); - out++; - *out = (inl >> 2) % (1U << 26); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 22)) << (26 - 22); - out++; - *out = (inl >> 22); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (26 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 10)) << (26 - 10); - out++; - *out = (inl >> 10); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (26 - 4); - out++; - *out = (inl >> 4) % (1U << 26); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 24)) << (26 - 24); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 18)) << (26 - 18); - out++; - *out = (inl >> 18); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (26 - 12); - out++; - *out = (inl >> 12); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (26 - 6); - out++; - *out = (inl >> 6); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack27_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 27); - out++; - *out = (inl >> 27); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 22)) << (27 - 22); - out++; - *out = (inl >> 22); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 17)) << (27 - 17); - out++; - *out = (inl >> 17); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (27 - 12); - out++; - *out = (inl >> 12); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 7)) << (27 - 7); - out++; - *out = (inl >> 7); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (27 - 2); - out++; - *out = (inl >> 2) % (1U << 27); - out++; - *out = (inl >> 29); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 24)) << (27 - 24); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 19)) << (27 - 19); - out++; - *out = (inl >> 19); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 14)) << (27 - 14); - out++; - *out = (inl >> 14); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 9)) << (27 - 9); - out++; - *out = (inl >> 9); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (27 - 4); - out++; - *out = (inl >> 4) % (1U << 27); - out++; - *out = (inl >> 31); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 26)) << (27 - 26); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 21)) << (27 - 21); - out++; - *out = (inl >> 21); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (27 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 11)) << (27 - 11); - out++; - *out = (inl >> 11); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (27 - 6); - out++; - *out = (inl >> 6); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 1)) << (27 - 1); - out++; - *out = (inl >> 1) % (1U << 27); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 23)) << (27 - 23); - out++; - *out = (inl >> 23); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 18)) << (27 - 18); - out++; - *out = (inl >> 18); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 13)) << (27 - 13); - out++; - *out = (inl >> 13); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (27 - 8); - out++; - *out = (inl >> 8); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 3)) << (27 - 3); - out++; - *out = (inl >> 3) % (1U << 27); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 25)) << (27 - 25); - out++; - *out = (inl >> 25); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 20)) << (27 - 20); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 15)) << (27 - 15); - out++; - *out = (inl >> 15); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 10)) << (27 - 10); - out++; - *out = (inl >> 10); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 5)) << (27 - 5); - out++; - *out = (inl >> 5); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack28_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 28); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 24)) << (28 - 24); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 20)) << (28 - 20); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (28 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (28 - 12); - out++; - *out = (inl >> 12); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (28 - 8); - out++; - *out = (inl >> 8); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (28 - 4); - out++; - *out = (inl >> 4); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 28); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 24)) << (28 - 24); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 20)) << (28 - 20); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (28 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (28 - 12); - out++; - *out = (inl >> 12); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (28 - 8); - out++; - *out = (inl >> 8); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (28 - 4); - out++; - *out = (inl >> 4); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 28); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 24)) << (28 - 24); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 20)) << (28 - 20); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (28 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (28 - 12); - out++; - *out = (inl >> 12); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (28 - 8); - out++; - *out = (inl >> 8); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (28 - 4); - out++; - *out = (inl >> 4); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 28); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 24)) << (28 - 24); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 20)) << (28 - 20); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (28 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (28 - 12); - out++; - *out = (inl >> 12); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (28 - 8); - out++; - *out = (inl >> 8); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (28 - 4); - out++; - *out = (inl >> 4); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack29_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 29); - out++; - *out = (inl >> 29); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 26)) << (29 - 26); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 23)) << (29 - 23); - out++; - *out = (inl >> 23); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 20)) << (29 - 20); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 17)) << (29 - 17); - out++; - *out = (inl >> 17); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 14)) << (29 - 14); - out++; - *out = (inl >> 14); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 11)) << (29 - 11); - out++; - *out = (inl >> 11); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (29 - 8); - out++; - *out = (inl >> 8); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 5)) << (29 - 5); - out++; - *out = (inl >> 5); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (29 - 2); - out++; - *out = (inl >> 2) % (1U << 29); - out++; - *out = (inl >> 31); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 28)) << (29 - 28); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 25)) << (29 - 25); - out++; - *out = (inl >> 25); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 22)) << (29 - 22); - out++; - *out = (inl >> 22); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 19)) << (29 - 19); - out++; - *out = (inl >> 19); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (29 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 13)) << (29 - 13); - out++; - *out = (inl >> 13); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 10)) << (29 - 10); - out++; - *out = (inl >> 10); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 7)) << (29 - 7); - out++; - *out = (inl >> 7); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (29 - 4); - out++; - *out = (inl >> 4); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 1)) << (29 - 1); - out++; - *out = (inl >> 1) % (1U << 29); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 27)) << (29 - 27); - out++; - *out = (inl >> 27); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 24)) << (29 - 24); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 21)) << (29 - 21); - out++; - *out = (inl >> 21); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 18)) << (29 - 18); - out++; - *out = (inl >> 18); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 15)) << (29 - 15); - out++; - *out = (inl >> 15); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (29 - 12); - out++; - *out = (inl >> 12); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 9)) << (29 - 9); - out++; - *out = (inl >> 9); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (29 - 6); - out++; - *out = (inl >> 6); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 3)) << (29 - 3); - out++; - *out = (inl >> 3); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack30_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 30); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 28)) << (30 - 28); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 26)) << (30 - 26); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 24)) << (30 - 24); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 22)) << (30 - 22); - out++; - *out = (inl >> 22); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 20)) << (30 - 20); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 18)) << (30 - 18); - out++; - *out = (inl >> 18); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (30 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 14)) << (30 - 14); - out++; - *out = (inl >> 14); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (30 - 12); - out++; - *out = (inl >> 12); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 10)) << (30 - 10); - out++; - *out = (inl >> 10); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (30 - 8); - out++; - *out = (inl >> 8); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (30 - 6); - out++; - *out = (inl >> 6); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (30 - 4); - out++; - *out = (inl >> 4); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (30 - 2); - out++; - *out = (inl >> 2); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0) % (1U << 30); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 28)) << (30 - 28); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 26)) << (30 - 26); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 24)) << (30 - 24); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 22)) << (30 - 22); - out++; - *out = (inl >> 22); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 20)) << (30 - 20); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 18)) << (30 - 18); - out++; - *out = (inl >> 18); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (30 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 14)) << (30 - 14); - out++; - *out = (inl >> 14); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (30 - 12); - out++; - *out = (inl >> 12); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 10)) << (30 - 10); - out++; - *out = (inl >> 10); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (30 - 8); - out++; - *out = (inl >> 8); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (30 - 6); - out++; - *out = (inl >> 6); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (30 - 4); - out++; - *out = (inl >> 4); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (30 - 2); - out++; - *out = (inl >> 2); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack31_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0) % (1U << 31); - out++; - *out = (inl >> 31); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 30)) << (31 - 30); - out++; - *out = (inl >> 30); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 29)) << (31 - 29); - out++; - *out = (inl >> 29); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 28)) << (31 - 28); - out++; - *out = (inl >> 28); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 27)) << (31 - 27); - out++; - *out = (inl >> 27); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 26)) << (31 - 26); - out++; - *out = (inl >> 26); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 25)) << (31 - 25); - out++; - *out = (inl >> 25); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 24)) << (31 - 24); - out++; - *out = (inl >> 24); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 23)) << (31 - 23); - out++; - *out = (inl >> 23); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 22)) << (31 - 22); - out++; - *out = (inl >> 22); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 21)) << (31 - 21); - out++; - *out = (inl >> 21); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 20)) << (31 - 20); - out++; - *out = (inl >> 20); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 19)) << (31 - 19); - out++; - *out = (inl >> 19); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 18)) << (31 - 18); - out++; - *out = (inl >> 18); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 17)) << (31 - 17); - out++; - *out = (inl >> 17); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 16)) << (31 - 16); - out++; - *out = (inl >> 16); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 15)) << (31 - 15); - out++; - *out = (inl >> 15); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 14)) << (31 - 14); - out++; - *out = (inl >> 14); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 13)) << (31 - 13); - out++; - *out = (inl >> 13); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 12)) << (31 - 12); - out++; - *out = (inl >> 12); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 11)) << (31 - 11); - out++; - *out = (inl >> 11); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 10)) << (31 - 10); - out++; - *out = (inl >> 10); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 9)) << (31 - 9); - out++; - *out = (inl >> 9); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 8)) << (31 - 8); - out++; - *out = (inl >> 8); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 7)) << (31 - 7); - out++; - *out = (inl >> 7); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 6)) << (31 - 6); - out++; - *out = (inl >> 6); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 5)) << (31 - 5); - out++; - *out = (inl >> 5); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 4)) << (31 - 4); - out++; - *out = (inl >> 4); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 3)) << (31 - 3); - out++; - *out = (inl >> 3); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 2)) << (31 - 2); - out++; - *out = (inl >> 2); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out |= (inl % (1U << 1)) << (31 - 1); - out++; - *out = (inl >> 1); - ++in; - out++; - - return in; -} - -inline const uint32_t* unpack32_32(const uint32_t* in, uint32_t* out) { - uint32_t inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - inl = util::SafeLoad(in); - inl = arrow::bit_util::FromLittleEndian(inl); - out++; - *out = (inl >> 0); - ++in; - out++; - - return in; -} - -inline const uint32_t* nullunpacker32(const uint32_t* in, uint32_t* out) { - for (int k = 0; k < 32; ++k) { - out[k] = 0; - } - return in; -} - -} // namespace internal -} // namespace arrow diff --git a/cpp/src/arrow/util/bpacking_dispatch_internal.h b/cpp/src/arrow/util/bpacking_dispatch_internal.h new file mode 100644 index 0000000000000..d7eefb6cb1edf --- /dev/null +++ b/cpp/src/arrow/util/bpacking_dispatch_internal.h @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "arrow/util/endian.h" +#include "arrow/util/logging.h" +#include "arrow/util/ubsan.h" + +namespace arrow::internal { + +/// Unpack a zero bit packed array. +template +int unpack_null(const uint8_t* in, Uint* out, int batch_size) { + std::memset(out, 0, batch_size * sizeof(Uint)); + return batch_size; +} + +/// Unpack a packed array where packed and unpacked values have exactly the same number of +/// bits. +template +int unpack_full(const uint8_t* in, Uint* out, int batch_size) { + if constexpr (ARROW_LITTLE_ENDIAN == 1) { + std::memcpy(out, in, batch_size * sizeof(Uint)); + } else { + using bit_util::FromLittleEndian; + using util::SafeLoadAs; + + for (int k = 0; k < batch_size; k += 1) { + out[k] = FromLittleEndian(SafeLoadAs(in + (k * sizeof(Uint)))); + } + } + return batch_size; +} + +/// Unpack a packed array, delegating to a Unpacker struct. +/// +/// @tparam kPackedBitWidth The width in bits of the values in the packed array. +/// @tparam Unpacker The struct providing information and an ``unpack`` method to unpack a +/// fixed amount of values (usually constrained by SIMD batch sizes and +/// byte alignment). +/// @tparam UnpackedUInt The type in which we unpack the values. +template typename Unpacker, + typename UnpackedUInt> +int unpack_width(const uint8_t* in, UnpackedUInt* out, int batch_size) { + using UnpackerForWidth = Unpacker; + + constexpr auto kValuesUnpacked = UnpackerForWidth::kValuesUnpacked; + batch_size = batch_size / kValuesUnpacked * kValuesUnpacked; + int num_loops = batch_size / kValuesUnpacked; + + for (int i = 0; i < num_loops; ++i) { + in = UnpackerForWidth::unpack(in, out + i * kValuesUnpacked); + } + + return batch_size; +} + +template