From 18bd5e51ba4a851ac46658f9ad5622d8355f6f43 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 19 Jun 2026 14:40:57 +0100 Subject: [PATCH 1/2] Try and improve some noise from cuda builds on macos Signed-off-by: Adam Gutglick --- vortex-cuda/build.rs | 86 +- vortex-cuda/kernels/src/bit_unpack_16.cu | 151 - .../kernels/src/bit_unpack_16_lanes.cuh | 867 -- vortex-cuda/kernels/src/bit_unpack_32.cu | 263 - .../kernels/src/bit_unpack_32_lanes.cuh | 3235 ---- vortex-cuda/kernels/src/bit_unpack_64.cu | 487 - .../kernels/src/bit_unpack_64_lanes.cuh | 12579 ---------------- vortex-cuda/kernels/src/bit_unpack_8.cu | 95 - .../kernels/src/bit_unpack_8_lanes.cuh | 259 - vortex-cuda/nvcomp/build.rs | 118 +- vortex-cuda/nvcomp/src/lib.rs | 17 +- 11 files changed, 163 insertions(+), 17994 deletions(-) delete mode 100644 vortex-cuda/kernels/src/bit_unpack_16.cu delete mode 100644 vortex-cuda/kernels/src/bit_unpack_16_lanes.cuh delete mode 100644 vortex-cuda/kernels/src/bit_unpack_32.cu delete mode 100644 vortex-cuda/kernels/src/bit_unpack_32_lanes.cuh delete mode 100644 vortex-cuda/kernels/src/bit_unpack_64.cu delete mode 100644 vortex-cuda/kernels/src/bit_unpack_64_lanes.cuh delete mode 100644 vortex-cuda/kernels/src/bit_unpack_8.cu delete mode 100644 vortex-cuda/kernels/src/bit_unpack_8_lanes.cuh diff --git a/vortex-cuda/build.rs b/vortex-cuda/build.rs index eb4743bab21..b2bb0afe6e8 100644 --- a/vortex-cuda/build.rs +++ b/vortex-cuda/build.rs @@ -21,16 +21,21 @@ use crate::bit_unpack_gen::generate_cuda_unpack_lanes; pub mod bit_unpack_gen; fn main() { - let manifest_dir = env::var("CARGO_MANIFEST_DIR").expect("Failed to get manifest dir"); + let manifest_dir = + PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("Failed to get manifest dir")); // https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-build-scripts let profile = env::var("PROFILE").unwrap(); + let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR not set")); - // Source directory for kernels (hand-written and generated .cu/.cuh files) - let kernels_src = Path::new(&manifest_dir).join("kernels/src"); - // Output directory for compiled .ptx files - separate by profile. - let kernels_gen = Path::new(&manifest_dir).join("kernels/gen").join(&profile); + // Source directory for hand-written kernels. + let kernels_src = manifest_dir.join("kernels/src"); + // Generated CUDA source and PTX are build artifacts and must stay under OUT_DIR. + let generated_kernels_src = out_dir.join("generated-kernels/src"); + let kernels_gen = out_dir.join("kernels").join(&profile); - std::fs::create_dir_all(&kernels_gen).expect("Failed to create kernels/gen directory"); + std::fs::create_dir_all(&kernels_gen).expect("Failed to create kernels directory"); + std::fs::create_dir_all(&generated_kernels_src) + .expect("Failed to create generated kernels directory"); // Always emit the kernels output directory path as a compile-time env var so any binary // linking against vortex-cuda can find the PTX files. This must be set regardless @@ -46,17 +51,20 @@ fn main() { // Regenerate bit_unpack kernels only when the generator changes println!( "cargo:rerun-if-changed={}", - Path::new(&manifest_dir) - .join("src/bit_unpack_gen.rs") - .display() + manifest_dir.join("src/bit_unpack_gen.rs").display() ); - generate_unpack::(&kernels_src, 32).expect("Failed to generate unpack for u8"); - generate_unpack::(&kernels_src, 32).expect("Failed to generate unpack for u16"); - generate_unpack::(&kernels_src, 32).expect("Failed to generate unpack for u32"); - generate_unpack::(&kernels_src, 16).expect("Failed to generate unpack for u64"); - - let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR not set")); - generate_arrow_device_array_bindings(Path::new(&manifest_dir), &out_dir); + let generated_cuda_sources = [ + generate_unpack::(&generated_kernels_src, 32) + .expect("Failed to generate unpack for u8"), + generate_unpack::(&generated_kernels_src, 32) + .expect("Failed to generate unpack for u16"), + generate_unpack::(&generated_kernels_src, 32) + .expect("Failed to generate unpack for u32"), + generate_unpack::(&generated_kernels_src, 16) + .expect("Failed to generate unpack for u64"), + ]; + + generate_arrow_device_array_bindings(&manifest_dir, &out_dir); generate_dynamic_dispatch_bindings(&kernels_src, &out_dir); generate_patches_bindings(&kernels_src, &out_dir); @@ -64,30 +72,18 @@ fn main() { return; } - // Watch and compile .cu and .cuh files from kernels/src to PTX in kernels/gen + let include_dirs = [kernels_src.as_path(), generated_kernels_src.as_path()]; + + // Watch and compile hand-written .cu and .cuh files from kernels/src. if let Ok(entries) = std::fs::read_dir(&kernels_src) { for path in entries.flatten().map(|entry| entry.path()) { - let is_generated = path - .file_name() - .and_then(|n| n.to_str()) - .is_some_and(|n| n.starts_with("bit_unpack_")); - match path.extension().and_then(|e| e.to_str()) { Some("cuh") | Some("h") => { - // Only watch hand-written .cuh/.h files, not generated ones - // (generated files are rebuilt when cuda_kernel_generator changes) - if !is_generated { - println!("cargo:rerun-if-changed={}", path.display()); - } + println!("cargo:rerun-if-changed={}", path.display()); } Some("cu") => { - // Only watch hand-written .cu files, not generated ones - // (generated files are rebuilt when cuda_kernel_generator changes) - if !is_generated { - println!("cargo:rerun-if-changed={}", path.display()); - } - // Compile all .cu files to PTX in gen directory - nvcc_compile_ptx(&kernels_src, &kernels_gen, &path, &profile) + println!("cargo:rerun-if-changed={}", path.display()); + nvcc_compile_ptx(&include_dirs, &kernels_gen, &path, &profile) .map_err(|e| { format!("Failed to compile CUDA kernel {}: {}", path.display(), e) }) @@ -97,6 +93,12 @@ fn main() { } } } + + for path in &generated_cuda_sources { + nvcc_compile_ptx(&include_dirs, &kernels_gen, path, &profile) + .map_err(|e| format!("Failed to compile CUDA kernel {}: {}", path.display(), e)) + .unwrap(); + } } fn generate_unpack(output_dir: &Path, thread_count: usize) -> io::Result { @@ -116,7 +118,7 @@ fn generate_unpack(output_dir: &Path, thread_count: usize) -> io:: } fn nvcc_compile_ptx( - include_dir: &Path, + include_dirs: &[&Path; 2], output_dir: &Path, cu_path: &Path, profile: &str, @@ -158,13 +160,13 @@ fn nvcc_compile_ptx( // Flags forwarded to Clang. .arg("--compiler-options=-Wall -Wextra -Wpedantic -Werror") .arg("--restrict") - .arg("--ptx") - .arg("--include-path") - .arg(include_dir) - .arg("-c") - .arg(cu_path) - .arg("-o") - .arg(&ptx_path); + .arg("--ptx"); + + for include_dir in include_dirs { + cmd.arg("--include-path").arg(include_dir); + } + + cmd.arg("-c").arg(cu_path).arg("-o").arg(&ptx_path); let res = cmd.output()?; diff --git a/vortex-cuda/kernels/src/bit_unpack_16.cu b/vortex-cuda/kernels/src/bit_unpack_16.cu deleted file mode 100644 index a784df201d3..00000000000 --- a/vortex-cuda/kernels/src/bit_unpack_16.cu +++ /dev/null @@ -1,151 +0,0 @@ -// AUTO-GENERATED. Do not edit by hand! -#include "bit_unpack_16_lanes.cuh" -#include "patches.cuh" - -template -__device__ void _bit_unpack_16_device(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, int thread_idx, GPUPatches& patches) { - __shared__ uint16_t shared_out[FL_CHUNK]; - - // Step 1: Unpack into shared memory - #pragma unroll - for (int i = 0; i < FL_LANES / 32; i++) { - _bit_unpack_16_lane(in, shared_out, reference, thread_idx * (FL_LANES / 32) + i); - } - __syncwarp(); - - // Step 2: Apply patches to shared memory in parallel - PatchesCursor cursor(patches, blockIdx.x, thread_idx, 32); - auto patch = cursor.next(); - while (patch.index != FL_CHUNK) { - shared_out[patch.index] = patch.value; - patch = cursor.next(); - } - __syncwarp(); - - // Step 3: Copy to global memory - #pragma unroll - for (int i = 0; i < FL_CHUNK / 32; i++) { - auto idx = i * 32 + thread_idx; - out[idx] = shared_out[idx]; - } -} - -extern "C" __global__ void bit_unpack_16_0bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 0)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_16_device<0>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_16_1bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 1)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_16_device<1>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_16_2bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 2)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_16_device<2>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_16_3bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 3)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_16_device<3>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_16_4bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 4)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_16_device<4>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_16_5bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 5)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_16_device<5>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_16_6bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 6)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_16_device<6>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_16_7bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 7)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_16_device<7>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_16_8bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 8)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_16_device<8>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_16_9bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 9)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_16_device<9>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_16_10bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 10)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_16_device<10>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_16_11bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 11)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_16_device<11>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_16_12bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 12)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_16_device<12>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_16_13bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 13)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_16_device<13>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_16_14bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 14)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_16_device<14>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_16_15bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 15)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_16_device<15>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_16_16bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 16)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_16_device<16>(in, out, reference, thread_idx, patches); -} - diff --git a/vortex-cuda/kernels/src/bit_unpack_16_lanes.cuh b/vortex-cuda/kernels/src/bit_unpack_16_lanes.cuh deleted file mode 100644 index ff992fa4ffb..00000000000 --- a/vortex-cuda/kernels/src/bit_unpack_16_lanes.cuh +++ /dev/null @@ -1,867 +0,0 @@ -// AUTO-GENERATED. Do not edit by hand! -#pragma once - -#include -#include -#include -#include "fastlanes_common.cuh" - -template -__device__ void _bit_unpack_16_lane(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane); - -template <> -__device__ void _bit_unpack_16_lane<0>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - #pragma unroll - for (int row = 0; row < 16; row++) { - out[INDEX(row, lane)] = reference; - } -} - -template <> -__device__ void _bit_unpack_16_lane<1>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 1); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint16_t, 1); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 1); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint16_t, 1); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 1); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint16_t, 1); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 1); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint16_t, 1); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 1); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint16_t, 1); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 1); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint16_t, 1); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 1); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint16_t, 1); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 1); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint16_t, 1); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<2>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 2); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 2); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 2); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 2); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 2); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 2); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 2); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 0)) << 2; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 2); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 2); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 2); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 2); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 2); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 2); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 2); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<3>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 3); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint16_t, 3); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 3); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint16_t, 3); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 3); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint16_t, 1); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 2)) << 1; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 3); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint16_t, 3); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 3); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint16_t, 3); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 1)) << 2; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint16_t, 3); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 3); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint16_t, 3); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 3); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint16_t, 3); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<4>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 4); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 4); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 4); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 0)) << 4; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 4); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 4); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 4); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 0)) << 4; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 4); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 4); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 4); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 0)) << 4; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 4); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 4); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 4); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<5>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 5); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint16_t, 5); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 5); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint16_t, 1); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 4)) << 1; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 5); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint16_t, 5); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 3)) << 2; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint16_t, 5); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 5); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint16_t, 3); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 2)) << 3; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 5); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint16_t, 5); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint16_t, 1)) << 4; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint16_t, 5); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 5); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint16_t, 5); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<6>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 6); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 6); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 2)) << 4; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 6); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 6); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 4)) << 2; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 6); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 0)) << 6; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 6); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 6); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint16_t, 2)) << 4; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 6); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 6); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint16_t, 4)) << 2; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 6); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 6); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<7>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 7); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint16_t, 7); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 5)) << 2; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint16_t, 7); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 3)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint16_t, 7); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 1)) << 6; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint16_t, 7); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 7); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint16_t, 1); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint16_t, 6)) << 1; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 7); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint16_t, 3); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint16_t, 4)) << 3; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 7); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint16_t, 5); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint16_t, 2)) << 5; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 7); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint16_t, 7); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<8>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 8); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 0)) << 8; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 8); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 0)) << 8; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 8); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 0)) << 8; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 8); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint16_t, 0)) << 8; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 8); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint16_t, 0)) << 8; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 8); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint16_t, 0)) << 8; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 8); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint16_t, 0)) << 8; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 8); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<9>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 9); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint16_t, 7); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 2)) << 7; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 9); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint16_t, 5); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 4)) << 5; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 9); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint16_t, 3); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 6)) << 3; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 9); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint16_t, 1); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint16_t, 8)) << 1; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint16_t, 1)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint16_t, 9); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint16_t, 3)) << 6; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint16_t, 9); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint16_t, 5)) << 4; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint16_t, 9); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint16_t, 7)) << 2; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint16_t, 9); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<10>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 10); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 6); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 4)) << 6; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 10); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 8)) << 2; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 2)) << 8; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 10); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint16_t, 6)) << 4; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 10); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint16_t, 0)) << 10; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 10); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint16_t, 4)) << 6; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 10); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint16_t, 8)) << 2; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint16_t, 2)) << 8; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 10); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint16_t, 6)) << 4; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 10); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<11>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 11); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint16_t, 5); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 6)) << 5; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 10); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 1)) << 10; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint16_t, 11); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 7)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint16_t, 9); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint16_t, 2)) << 9; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 11); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint16_t, 3); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint16_t, 8)) << 3; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint16_t, 3)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint16_t, 11); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint16_t, 9)) << 2; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint16_t, 7); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint16_t, 4)) << 7; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 11); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint16_t, 1); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint16_t, 10)) << 1; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 6); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint16_t, 5)) << 6; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint16_t, 11); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<12>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 12); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 8)) << 4; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 4)) << 8; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 0)) << 12; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 12); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint16_t, 8)) << 4; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint16_t, 4)) << 8; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 12); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint16_t, 0)) << 12; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 12); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint16_t, 8)) << 4; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint16_t, 4)) << 8; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 12); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint16_t, 0)) << 12; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 12); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint16_t, 8)) << 4; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint16_t, 4)) << 8; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 12); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<13>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 13); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint16_t, 3); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 10)) << 3; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 6); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 7)) << 6; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint16_t, 9); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 4)) << 9; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 12); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint16_t, 1)) << 12; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint16_t, 13); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint16_t, 11)) << 2; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint16_t, 5); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint16_t, 8)) << 5; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint16_t, 5)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint16_t, 11); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint16_t, 2)) << 11; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 13); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint16_t, 1); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint16_t, 12)) << 1; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint16_t, 9)) << 4; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint16_t, 7); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint16_t, 6)) << 7; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 10); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint16_t, 3)) << 10; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint16_t, 13); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<14>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 14); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 12)) << 2; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 10)) << 4; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 8)) << 6; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint16_t, 6)) << 8; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 10); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint16_t, 4)) << 10; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 12); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint16_t, 2)) << 12; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 14); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint16_t, 0)) << 14; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 14); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint16_t, 12)) << 2; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint16_t, 10)) << 4; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 6); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint16_t, 8)) << 6; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint16_t, 6)) << 8; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 10); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint16_t, 4)) << 10; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 12); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint16_t, 2)) << 12; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 14); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<15>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 15); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint16_t, 1); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 14)) << 1; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 13)) << 2; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint16_t, 3); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 12)) << 3; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint16_t, 11)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint16_t, 5); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint16_t, 10)) << 5; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint16_t, 9)) << 6; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint16_t, 7); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint16_t, 8)) << 7; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint16_t, 7)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint16_t, 9); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint16_t, 6)) << 9; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 10); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint16_t, 5)) << 10; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint16_t, 11); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint16_t, 4)) << 11; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 12); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint16_t, 3)) << 12; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint16_t, 13); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint16_t, 2)) << 13; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 14); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint16_t, 1)) << 14; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint16_t, 15); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<16>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - #pragma unroll - for (int row = 0; row < 16; row++) { - out[INDEX(row, lane)] = in[LANE_COUNT * row + lane] + reference; - } -} - -/// Runtime dispatch to the optimized lane decoder for the given bit width. -__device__ __noinline__ void bit_unpack_16_lane( - const uint16_t *__restrict in, - uint16_t *__restrict out, - uint16_t reference, - unsigned int lane, - uint32_t bit_width -) { - switch (bit_width) { - case 0: _bit_unpack_16_lane<0>(in, out, reference, lane); break; - case 1: _bit_unpack_16_lane<1>(in, out, reference, lane); break; - case 2: _bit_unpack_16_lane<2>(in, out, reference, lane); break; - case 3: _bit_unpack_16_lane<3>(in, out, reference, lane); break; - case 4: _bit_unpack_16_lane<4>(in, out, reference, lane); break; - case 5: _bit_unpack_16_lane<5>(in, out, reference, lane); break; - case 6: _bit_unpack_16_lane<6>(in, out, reference, lane); break; - case 7: _bit_unpack_16_lane<7>(in, out, reference, lane); break; - case 8: _bit_unpack_16_lane<8>(in, out, reference, lane); break; - case 9: _bit_unpack_16_lane<9>(in, out, reference, lane); break; - case 10: _bit_unpack_16_lane<10>(in, out, reference, lane); break; - case 11: _bit_unpack_16_lane<11>(in, out, reference, lane); break; - case 12: _bit_unpack_16_lane<12>(in, out, reference, lane); break; - case 13: _bit_unpack_16_lane<13>(in, out, reference, lane); break; - case 14: _bit_unpack_16_lane<14>(in, out, reference, lane); break; - case 15: _bit_unpack_16_lane<15>(in, out, reference, lane); break; - case 16: _bit_unpack_16_lane<16>(in, out, reference, lane); break; - } -} - diff --git a/vortex-cuda/kernels/src/bit_unpack_32.cu b/vortex-cuda/kernels/src/bit_unpack_32.cu deleted file mode 100644 index 3f8fcb5c227..00000000000 --- a/vortex-cuda/kernels/src/bit_unpack_32.cu +++ /dev/null @@ -1,263 +0,0 @@ -// AUTO-GENERATED. Do not edit by hand! -#include "bit_unpack_32_lanes.cuh" -#include "patches.cuh" - -template -__device__ void _bit_unpack_32_device(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, int thread_idx, GPUPatches& patches) { - __shared__ uint32_t shared_out[FL_CHUNK]; - - // Step 1: Unpack into shared memory - #pragma unroll - for (int i = 0; i < FL_LANES / 32; i++) { - _bit_unpack_32_lane(in, shared_out, reference, thread_idx * (FL_LANES / 32) + i); - } - __syncwarp(); - - // Step 2: Apply patches to shared memory in parallel - PatchesCursor cursor(patches, blockIdx.x, thread_idx, 32); - auto patch = cursor.next(); - while (patch.index != FL_CHUNK) { - shared_out[patch.index] = patch.value; - patch = cursor.next(); - } - __syncwarp(); - - // Step 3: Copy to global memory - #pragma unroll - for (int i = 0; i < FL_CHUNK / 32; i++) { - auto idx = i * 32 + thread_idx; - out[idx] = shared_out[idx]; - } -} - -extern "C" __global__ void bit_unpack_32_0bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 0)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<0>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_1bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 1)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<1>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_2bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 2)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<2>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_3bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 3)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<3>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_4bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 4)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<4>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_5bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 5)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<5>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_6bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 6)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<6>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_7bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 7)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<7>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_8bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 8)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<8>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_9bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 9)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<9>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_10bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 10)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<10>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_11bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 11)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<11>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_12bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 12)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<12>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_13bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 13)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<13>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_14bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 14)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<14>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_15bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 15)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<15>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_16bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 16)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<16>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_17bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 17)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<17>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_18bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 18)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<18>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_19bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 19)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<19>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_20bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 20)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<20>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_21bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 21)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<21>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_22bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 22)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<22>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_23bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 23)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<23>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_24bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 24)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<24>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_25bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 25)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<25>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_26bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 26)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<26>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_27bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 27)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<27>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_28bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 28)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<28>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_29bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 29)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<29>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_30bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 30)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<30>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_31bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 31)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<31>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_32_32bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 32)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_32_device<32>(in, out, reference, thread_idx, patches); -} - diff --git a/vortex-cuda/kernels/src/bit_unpack_32_lanes.cuh b/vortex-cuda/kernels/src/bit_unpack_32_lanes.cuh deleted file mode 100644 index 58d4195e3aa..00000000000 --- a/vortex-cuda/kernels/src/bit_unpack_32_lanes.cuh +++ /dev/null @@ -1,3235 +0,0 @@ -// AUTO-GENERATED. Do not edit by hand! -#pragma once - -#include -#include -#include -#include "fastlanes_common.cuh" - -template -__device__ void _bit_unpack_32_lane(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane); - -template <> -__device__ void _bit_unpack_32_lane<0>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - #pragma unroll - for (int row = 0; row < 32; row++) { - out[INDEX(row, lane)] = reference; - } -} - -template <> -__device__ void _bit_unpack_32_lane<1>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 1); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 1); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 1); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 1); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 1); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 1); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 1); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 1); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 1); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 1); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 1); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 1); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 1); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 1); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 1); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 1); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 1); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 1); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 1); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 1); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 1); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 1); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 1); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 1); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 1); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 1); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 1); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 1); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 1); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 1); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 1); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<2>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 2); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 2); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 2); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 2); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 2); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 2); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 2); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 2); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 2); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 2); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 2); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 2); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 2); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 2); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 2); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 0)) << 2; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 2); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 2); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 2); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 2); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 2); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 2); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 2); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 2); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 2); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 2); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 2); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 2); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 2); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 2); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 2); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<3>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 3); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 3); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 3); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 3); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 3); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 3); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 3); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 3); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 3); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 3); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 1)) << 2; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 3); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 3); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 3); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 3); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 3); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 3); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 3); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 3); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 3); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 3); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 2)) << 1; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 3); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 3); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 3); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 3); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 3); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 3); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 3); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 3); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 3); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<4>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 4); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 4); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 4); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 4); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 4); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 4); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 4); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 0)) << 4; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 4); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 4); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 4); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 4); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 4); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 4); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 4); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 0)) << 4; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 4); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 4); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 4); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 4); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 4); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 4); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 4); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 0)) << 4; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 4); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 4); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 4); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 4); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 4); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 4); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 4); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<5>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 5); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 5); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 5); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 5); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 5); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 5); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 3)) << 2; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 5); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 5); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 5); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 5); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 5); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 1)) << 4; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 5); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 5); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 5); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 5); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 5); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 5); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 4)) << 1; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 5); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 5); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 5); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 5); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 5); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 2)) << 3; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 5); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 5); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 5); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 5); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 5); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<6>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 6); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 6); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 6); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 6); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 6); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 4)) << 2; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 6); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 6); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 6); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 6); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 2)) << 4; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 6); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 6); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 6); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 6); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 0)) << 6; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 6); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 6); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 6); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 6); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 6); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 4)) << 2; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 6); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 6); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 6); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 6); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 2)) << 4; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 6); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 6); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 6); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 6); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<7>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 7); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 7); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 7); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 7); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 3)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 7); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 7); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 7); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 7); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 6)) << 1; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 7); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 7); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 7); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 2)) << 5; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 7); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 7); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 7); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 7); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 5)) << 2; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 7); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 7); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 7); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 1)) << 6; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 7); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 7); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 7); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 7); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 4)) << 3; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 7); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 7); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 7); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<8>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 8); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 8); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 8); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 0)) << 8; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 8); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 8); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 8); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 0)) << 8; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 8); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 8); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 8); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 0)) << 8; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 8); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 8); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 8); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 0)) << 8; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 8); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 8); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 8); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 0)) << 8; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 8); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 8); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 8); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 0)) << 8; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 8); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 8); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 8); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 0)) << 8; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 8); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 8); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 8); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<9>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 9); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 9); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 9); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 4)) << 5; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 9); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 9); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 9); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 8)) << 1; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 9); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 9); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 3)) << 6; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 9); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 9); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 9); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 7)) << 2; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 9); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 9); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 2)) << 7; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 9); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 9); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 9); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 6)) << 3; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 9); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 9); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 1)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 9); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 9); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 9); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 5)) << 4; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 9); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 9); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<10>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 10); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 10); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 10); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 8)) << 2; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 10); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 10); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 6)) << 4; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 10); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 10); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 4)) << 6; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 10); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 10); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 2)) << 8; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 10); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 10); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 0)) << 10; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 10); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 10); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 10); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 8)) << 2; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 10); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 10); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 6)) << 4; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 10); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 10); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 4)) << 6; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 10); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 10); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 2)) << 8; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 10); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 10); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<11>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 11); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 11); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 1)) << 10; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 11); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 11); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 2)) << 9; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 11); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 11); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 3)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 11); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 11); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 4)) << 7; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 11); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 11); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 5)) << 6; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 11); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 11); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 6)) << 5; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 11); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 11); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 7)) << 4; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 11); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 11); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 8)) << 3; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 11); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 11); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 9)) << 2; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 11); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 11); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 10)) << 1; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 11); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 11); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<12>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 12); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 12); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 4)) << 8; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 12); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 12); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 8)) << 4; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 12); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 0)) << 12; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 12); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 12); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 4)) << 8; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 12); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 12); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 8)) << 4; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 12); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 0)) << 12; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 12); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 12); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 4)) << 8; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 12); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 12); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 8)) << 4; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 12); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 0)) << 12; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 12); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 12); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 4)) << 8; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 12); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 12); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 8)) << 4; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 12); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<13>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 13); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 13); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 7)) << 6; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 13); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 1)) << 12; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 13); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 13); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 8)) << 5; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 13); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 11); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 2)) << 11; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 13); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 13); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 9)) << 4; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 13); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 3)) << 10; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 13); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 13); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 10)) << 3; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 13); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 4)) << 9; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 13); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 13); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 11)) << 2; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 13); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 5)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 13); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 13); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 12)) << 1; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 13); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 6)) << 7; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 13); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 13); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<14>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 14); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 14); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 10)) << 4; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 14); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 6)) << 8; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 14); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 2)) << 12; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 14); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 14); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 12)) << 2; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 14); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 8)) << 6; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 14); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 4)) << 10; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 14); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 0)) << 14; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 14); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 14); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 10)) << 4; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 14); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 6)) << 8; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 14); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 2)) << 12; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 14); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 14); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 12)) << 2; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 14); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 8)) << 6; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 14); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 4)) << 10; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 14); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<15>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 15); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 15); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 13)) << 2; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 15); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 11)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 15); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 9)) << 6; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 15); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 7)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 15); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 5)) << 10; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 15); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 3)) << 12; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 15); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 1)) << 14; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 15); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 15); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 14)) << 1; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 15); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 12)) << 3; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 15); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 10)) << 5; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 15); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 8)) << 7; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 15); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 6)) << 9; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 15); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 11); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 4)) << 11; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 15); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 13); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 2)) << 13; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 15); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 15); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<16>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<17>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 17); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 15); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 2)) << 15; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 17); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 13); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 4)) << 13; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 17); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 11); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 6)) << 11; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 17); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 8)) << 9; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 17); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 10)) << 7; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 17); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 12)) << 5; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 17); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 14)) << 3; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 17); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 16)) << 1; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 1)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 17); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 3)) << 14; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 17); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 5)) << 12; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 17); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 7)) << 10; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 17); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 9)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 17); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 11)) << 6; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 17); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 13)) << 4; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 17); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 15)) << 2; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 17); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<18>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 18); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 4)) << 14; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 18); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 8)) << 10; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 18); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 12)) << 6; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 18); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 16)) << 2; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 2)) << 16; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 18); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 6)) << 12; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 18); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 10)) << 8; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 18); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 14)) << 4; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 0)) << 18; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 18); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 4)) << 14; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 18); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 8)) << 10; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 18); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 12)) << 6; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 18); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 16)) << 2; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 2)) << 16; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 18); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 6)) << 12; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 18); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 10)) << 8; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 18); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 14)) << 4; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<19>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 19); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 13); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 6)) << 13; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 19); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 12)) << 7; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 19); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 18)) << 1; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 5)) << 14; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 19); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 11)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 19); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 17)) << 2; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 15); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 4)) << 15; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 19); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 10)) << 9; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 19); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 16)) << 3; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 3)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 19); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 9)) << 10; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 19); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 15)) << 4; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 17); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 2)) << 17; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 19); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 11); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 8)) << 11; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 19); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 14)) << 5; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 1)) << 18; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 19); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 7)) << 12; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 19); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 13)) << 6; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 19); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<20>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 20); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 8)) << 12; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 20); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 16)) << 4; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 4)) << 16; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 20); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 12)) << 8; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 0)) << 20; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 20); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 8)) << 12; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 20); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 16)) << 4; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 4)) << 16; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 20); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 12)) << 8; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 0)) << 20; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 20); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 8)) << 12; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 20); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 16)) << 4; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 4)) << 16; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 20); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 12)) << 8; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 0)) << 20; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 20); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 8)) << 12; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 20); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 16)) << 4; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 4)) << 16; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 20); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 12)) << 8; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<21>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 21); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 11); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 10)) << 11; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 21); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 20)) << 1; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 9)) << 12; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 21); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 19)) << 2; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 13); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 8)) << 13; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 21); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 18)) << 3; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 7)) << 14; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 21); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 17)) << 4; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 15); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 6)) << 15; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 21); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 16)) << 5; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 5)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 21); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 15)) << 6; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 17); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 4)) << 17; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 21); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 14)) << 7; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 3)) << 18; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 21); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 13)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 19); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 2)) << 19; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 21); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 12)) << 9; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 1)) << 20; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 21); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint32_t, 11)) << 10; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 21); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<22>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 22); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 12)) << 10; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 2)) << 20; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 22); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 14)) << 8; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 4)) << 18; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 22); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 16)) << 6; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 6)) << 16; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 22); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 18)) << 4; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 8)) << 14; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 22); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 20)) << 2; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 10)) << 12; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 22); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 0)) << 22; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 22); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 12)) << 10; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 2)) << 20; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 22); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 14)) << 8; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 4)) << 18; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 22); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 16)) << 6; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 6)) << 16; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 22); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 18)) << 4; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 8)) << 14; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 22); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint32_t, 20)) << 2; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint32_t, 10)) << 12; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 22); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<23>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 23); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 14)) << 9; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 5)) << 18; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 23); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 19)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 13); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 10)) << 13; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 22); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 1)) << 22; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 23); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 15)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 17); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 6)) << 17; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 23); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 20)) << 3; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 11)) << 12; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 21); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 2)) << 21; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 23); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 16)) << 7; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 7)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 23); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 21)) << 2; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 11); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 12)) << 11; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 3)) << 20; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 23); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 17)) << 6; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 15); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 8)) << 15; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 23); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 22)) << 1; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 13)) << 10; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 19); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint32_t, 4)) << 19; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 23); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint32_t, 18)) << 5; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint32_t, 9)) << 14; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 23); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<24>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 24); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 16)) << 8; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 8)) << 16; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 0)) << 24; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 24); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 16)) << 8; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 8)) << 16; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 0)) << 24; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 24); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 16)) << 8; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 8)) << 16; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 0)) << 24; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 24); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 16)) << 8; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 8)) << 16; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 0)) << 24; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 24); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 16)) << 8; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 8)) << 16; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 0)) << 24; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 24); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 16)) << 8; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 8)) << 16; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 0)) << 24; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 24); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 16)) << 8; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint32_t, 8)) << 16; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint32_t, 0)) << 24; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 24); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint32_t, 16)) << 8; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint32_t, 8)) << 16; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<25>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 25); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 18)) << 7; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 11)) << 14; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 21); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 4)) << 21; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 25); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 22)) << 3; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 15)) << 10; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 17); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 8)) << 17; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 1)) << 24; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 25); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 19)) << 6; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 13); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 12)) << 13; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 5)) << 20; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 25); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 23)) << 2; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 16)) << 9; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 9)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 23); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 2)) << 23; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 25); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 20)) << 5; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 13)) << 12; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 19); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 6)) << 19; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 25); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 24)) << 1; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 17)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 15); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint32_t, 10)) << 15; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 22); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint32_t, 3)) << 22; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 25); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint32_t, 21)) << 4; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 11); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint32_t, 14)) << 11; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint32_t, 7)) << 18; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 25); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<26>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 26); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 20)) << 6; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 14)) << 12; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 8)) << 18; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 2)) << 24; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 26); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 22)) << 4; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 16)) << 10; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 10)) << 16; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 22); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 4)) << 22; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 26); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 24)) << 2; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 18)) << 8; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 12)) << 14; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 6)) << 20; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 26); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 0)) << 26; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 26); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 20)) << 6; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 14)) << 12; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 8)) << 18; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 2)) << 24; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 26); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 22)) << 4; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 16)) << 10; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint32_t, 10)) << 16; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 22); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint32_t, 4)) << 22; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 26); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint32_t, 24)) << 2; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint32_t, 18)) << 8; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint32_t, 12)) << 14; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint32_t, 6)) << 20; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 26); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<27>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 27); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 22)) << 5; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 17)) << 10; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 15); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 12)) << 15; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 7)) << 20; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 25); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 2)) << 25; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 27); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 24)) << 3; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 19)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 13); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 14)) << 13; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 9)) << 18; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 23); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 4)) << 23; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 27); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 26)) << 1; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 21)) << 6; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 11); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 16)) << 11; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 11)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 21); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 6)) << 21; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 26); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 1)) << 26; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 27); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 23)) << 4; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 18)) << 9; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 13)) << 14; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 19); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint32_t, 8)) << 19; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint32_t, 3)) << 24; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 27); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint32_t, 25)) << 2; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint32_t, 20)) << 7; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint32_t, 15)) << 12; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 17); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint32_t, 10)) << 17; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 22); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint32_t, 5)) << 22; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 27); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<28>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 28); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 24)) << 4; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 20)) << 8; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 16)) << 12; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 12)) << 16; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 8)) << 20; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 4)) << 24; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 28); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 0)) << 28; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 28); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 24)) << 4; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 20)) << 8; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 16)) << 12; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 12)) << 16; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 8)) << 20; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 4)) << 24; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 28); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 0)) << 28; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 28); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 24)) << 4; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 20)) << 8; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 16)) << 12; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 12)) << 16; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 8)) << 20; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint32_t, 4)) << 24; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 28); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint32_t, 0)) << 28; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 28); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint32_t, 24)) << 4; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint32_t, 20)) << 8; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint32_t, 16)) << 12; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint32_t, 12)) << 16; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint32_t, 8)) << 20; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint32_t, 4)) << 24; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 28); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<29>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 29); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 26)) << 3; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 23)) << 6; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 20)) << 9; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 17)) << 12; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 15); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 14)) << 15; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 11)) << 18; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 21); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 8)) << 21; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 5)) << 24; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 27); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 2)) << 27; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 29); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 28)) << 1; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 25)) << 4; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 22)) << 7; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 19)) << 10; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 13); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 16)) << 13; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 13)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 19); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 10)) << 19; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 22); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 7)) << 22; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 25); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 4)) << 25; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 28); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 1)) << 28; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 29); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint32_t, 27)) << 2; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint32_t, 24)) << 5; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint32_t, 21)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 11); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint32_t, 18)) << 11; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint32_t, 15)) << 14; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 17); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint32_t, 12)) << 17; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint32_t, 9)) << 20; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 23); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint32_t, 6)) << 23; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 26); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint32_t, 3)) << 26; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 29); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<30>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 30); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 28)) << 2; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 26)) << 4; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 24)) << 6; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 22)) << 8; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 20)) << 10; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 18)) << 12; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 16)) << 14; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 14)) << 16; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 12)) << 18; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 10)) << 20; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 22); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 8)) << 22; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 6)) << 24; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 26); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 4)) << 26; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 28); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 2)) << 28; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 30); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 0)) << 30; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 30); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 28)) << 2; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 26)) << 4; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 24)) << 6; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 22)) << 8; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint32_t, 20)) << 10; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint32_t, 18)) << 12; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint32_t, 16)) << 14; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint32_t, 14)) << 16; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint32_t, 12)) << 18; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint32_t, 10)) << 20; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 22); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint32_t, 8)) << 22; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint32_t, 6)) << 24; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 26); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint32_t, 4)) << 26; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 28); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint32_t, 2)) << 28; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 30); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<31>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 31); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 30)) << 1; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 29)) << 2; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 28)) << 3; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 27)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 26)) << 5; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 25)) << 6; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 24)) << 7; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 23)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 22)) << 9; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 21)) << 10; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 11); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 20)) << 11; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 19)) << 12; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 13); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 18)) << 13; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 17)) << 14; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 15); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 16)) << 15; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 15)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 17); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 14)) << 17; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 13)) << 18; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 19); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 12)) << 19; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint32_t, 11)) << 20; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 21); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint32_t, 10)) << 21; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 22); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint32_t, 9)) << 22; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 23); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint32_t, 8)) << 23; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint32_t, 7)) << 24; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 25); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint32_t, 6)) << 25; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 26); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint32_t, 5)) << 26; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 27); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint32_t, 4)) << 27; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 28); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint32_t, 3)) << 28; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 29); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint32_t, 2)) << 29; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 30); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint32_t, 1)) << 30; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 31); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<32>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - #pragma unroll - for (int row = 0; row < 32; row++) { - out[INDEX(row, lane)] = in[LANE_COUNT * row + lane] + reference; - } -} - -/// Runtime dispatch to the optimized lane decoder for the given bit width. -__device__ __noinline__ void bit_unpack_32_lane( - const uint32_t *__restrict in, - uint32_t *__restrict out, - uint32_t reference, - unsigned int lane, - uint32_t bit_width -) { - switch (bit_width) { - case 0: _bit_unpack_32_lane<0>(in, out, reference, lane); break; - case 1: _bit_unpack_32_lane<1>(in, out, reference, lane); break; - case 2: _bit_unpack_32_lane<2>(in, out, reference, lane); break; - case 3: _bit_unpack_32_lane<3>(in, out, reference, lane); break; - case 4: _bit_unpack_32_lane<4>(in, out, reference, lane); break; - case 5: _bit_unpack_32_lane<5>(in, out, reference, lane); break; - case 6: _bit_unpack_32_lane<6>(in, out, reference, lane); break; - case 7: _bit_unpack_32_lane<7>(in, out, reference, lane); break; - case 8: _bit_unpack_32_lane<8>(in, out, reference, lane); break; - case 9: _bit_unpack_32_lane<9>(in, out, reference, lane); break; - case 10: _bit_unpack_32_lane<10>(in, out, reference, lane); break; - case 11: _bit_unpack_32_lane<11>(in, out, reference, lane); break; - case 12: _bit_unpack_32_lane<12>(in, out, reference, lane); break; - case 13: _bit_unpack_32_lane<13>(in, out, reference, lane); break; - case 14: _bit_unpack_32_lane<14>(in, out, reference, lane); break; - case 15: _bit_unpack_32_lane<15>(in, out, reference, lane); break; - case 16: _bit_unpack_32_lane<16>(in, out, reference, lane); break; - case 17: _bit_unpack_32_lane<17>(in, out, reference, lane); break; - case 18: _bit_unpack_32_lane<18>(in, out, reference, lane); break; - case 19: _bit_unpack_32_lane<19>(in, out, reference, lane); break; - case 20: _bit_unpack_32_lane<20>(in, out, reference, lane); break; - case 21: _bit_unpack_32_lane<21>(in, out, reference, lane); break; - case 22: _bit_unpack_32_lane<22>(in, out, reference, lane); break; - case 23: _bit_unpack_32_lane<23>(in, out, reference, lane); break; - case 24: _bit_unpack_32_lane<24>(in, out, reference, lane); break; - case 25: _bit_unpack_32_lane<25>(in, out, reference, lane); break; - case 26: _bit_unpack_32_lane<26>(in, out, reference, lane); break; - case 27: _bit_unpack_32_lane<27>(in, out, reference, lane); break; - case 28: _bit_unpack_32_lane<28>(in, out, reference, lane); break; - case 29: _bit_unpack_32_lane<29>(in, out, reference, lane); break; - case 30: _bit_unpack_32_lane<30>(in, out, reference, lane); break; - case 31: _bit_unpack_32_lane<31>(in, out, reference, lane); break; - case 32: _bit_unpack_32_lane<32>(in, out, reference, lane); break; - } -} - diff --git a/vortex-cuda/kernels/src/bit_unpack_64.cu b/vortex-cuda/kernels/src/bit_unpack_64.cu deleted file mode 100644 index ebe0b125369..00000000000 --- a/vortex-cuda/kernels/src/bit_unpack_64.cu +++ /dev/null @@ -1,487 +0,0 @@ -// AUTO-GENERATED. Do not edit by hand! -#include "bit_unpack_64_lanes.cuh" -#include "patches.cuh" - -template -__device__ void _bit_unpack_64_device(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, int thread_idx, GPUPatches& patches) { - __shared__ uint64_t shared_out[FL_CHUNK]; - - // Step 1: Unpack into shared memory - #pragma unroll - for (int i = 0; i < FL_LANES / 16; i++) { - _bit_unpack_64_lane(in, shared_out, reference, thread_idx * (FL_LANES / 16) + i); - } - __syncwarp(); - - // Step 2: Apply patches to shared memory in parallel - PatchesCursor cursor(patches, blockIdx.x, thread_idx, 16); - auto patch = cursor.next(); - while (patch.index != FL_CHUNK) { - shared_out[patch.index] = patch.value; - patch = cursor.next(); - } - __syncwarp(); - - // Step 3: Copy to global memory - #pragma unroll - for (int i = 0; i < FL_CHUNK / 16; i++) { - auto idx = i * 16 + thread_idx; - out[idx] = shared_out[idx]; - } -} - -extern "C" __global__ void bit_unpack_64_0bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 0)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<0>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_1bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 1)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<1>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_2bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 2)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<2>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_3bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 3)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<3>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_4bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 4)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<4>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_5bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 5)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<5>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_6bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 6)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<6>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_7bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 7)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<7>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_8bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 8)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<8>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_9bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 9)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<9>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_10bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 10)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<10>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_11bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 11)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<11>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_12bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 12)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<12>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_13bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 13)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<13>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_14bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 14)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<14>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_15bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 15)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<15>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_16bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 16)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<16>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_17bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 17)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<17>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_18bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 18)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<18>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_19bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 19)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<19>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_20bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 20)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<20>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_21bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 21)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<21>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_22bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 22)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<22>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_23bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 23)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<23>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_24bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 24)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<24>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_25bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 25)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<25>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_26bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 26)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<26>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_27bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 27)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<27>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_28bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 28)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<28>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_29bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 29)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<29>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_30bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 30)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<30>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_31bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 31)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<31>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_32bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 32)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<32>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_33bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 33)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<33>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_34bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 34)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<34>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_35bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 35)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<35>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_36bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 36)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<36>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_37bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 37)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<37>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_38bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 38)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<38>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_39bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 39)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<39>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_40bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 40)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<40>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_41bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 41)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<41>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_42bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 42)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<42>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_43bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 43)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<43>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_44bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 44)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<44>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_45bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 45)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<45>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_46bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 46)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<46>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_47bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 47)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<47>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_48bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 48)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<48>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_49bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 49)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<49>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_50bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 50)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<50>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_51bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 51)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<51>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_52bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 52)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<52>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_53bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 53)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<53>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_54bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 54)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<54>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_55bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 55)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<55>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_56bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 56)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<56>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_57bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 57)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<57>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_58bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 58)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<58>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_59bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 59)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<59>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_60bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 60)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<60>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_61bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 61)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<61>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_62bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 62)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<62>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_63bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 63)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<63>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_64_64bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 64)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_64_device<64>(in, out, reference, thread_idx, patches); -} - diff --git a/vortex-cuda/kernels/src/bit_unpack_64_lanes.cuh b/vortex-cuda/kernels/src/bit_unpack_64_lanes.cuh deleted file mode 100644 index afc17737b40..00000000000 --- a/vortex-cuda/kernels/src/bit_unpack_64_lanes.cuh +++ /dev/null @@ -1,12579 +0,0 @@ -// AUTO-GENERATED. Do not edit by hand! -#pragma once - -#include -#include -#include -#include "fastlanes_common.cuh" - -template -__device__ void _bit_unpack_64_lane(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane); - -template <> -__device__ void _bit_unpack_64_lane<0>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - #pragma unroll - for (int row = 0; row < 64; row++) { - out[INDEX(row, lane)] = reference; - } -} - -template <> -__device__ void _bit_unpack_64_lane<1>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 1); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 1); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 1); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 1); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 1); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 1); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 1); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 1); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 1); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 1); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 1); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 1); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 1); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 1); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 1); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 1); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 1); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 1); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 1); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 1); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 1); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 1); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 1); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 1); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 1); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 1); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 1); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 1); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 1); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 1); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 1); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 1); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 1); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 1); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 1); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 1); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 1); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 1); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 1); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 1); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 1); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 1); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 1); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 1); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 1); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 1); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 1); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 1); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 1); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 1); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 1); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 1); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 1); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 1); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 1); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 1); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 1); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 1); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 1); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 1); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 1); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 1); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 1); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<2>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 2); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 2); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 2); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 2); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 2); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 2); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 2); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 2); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 2); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 2); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 2); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 2); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 2); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 2); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 2); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 2); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 2); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 2); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 2); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 2); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 2); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 2); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 2); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 2); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 2); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 2); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 2); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 2); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 2); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 2); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 2); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 0)) << 2; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 2); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 2); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 2); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 2); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 2); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 2); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 2); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 2); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 2); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 2); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 2); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 2); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 2); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 2); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 2); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 2); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 2); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 2); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 2); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 2); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 2); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 2); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 2); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 2); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 2); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 2); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 2); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 2); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 2); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 2); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 2); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<3>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 3); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 3); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 3); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 3); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 3); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 3); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 3); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 3); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 3); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 3); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 3); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 3); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 3); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 3); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 3); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 3); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 3); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 3); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 3); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 3); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 3); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 2)) << 1; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 3); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 3); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 3); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 3); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 3); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 3); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 3); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 3); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 3); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 3); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 3); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 3); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 3); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 3); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 3); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 3); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 3); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 3); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 3); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 3); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 1)) << 2; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 3); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 3); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 3); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 3); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 3); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 3); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 3); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 3); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 3); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 3); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 3); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 3); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 3); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 3); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 3); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 3); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 3); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 3); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 3); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 3); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<4>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 4); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 4); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 4); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 4); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 4); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 4); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 4); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 4); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 4); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 4); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 4); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 4); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 4); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 4); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 4); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 0)) << 4; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 4); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 4); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 4); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 4); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 4); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 4); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 4); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 4); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 4); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 4); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 4); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 4); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 4); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 4); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 4); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 0)) << 4; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 4); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 4); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 4); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 4); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 4); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 4); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 4); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 4); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 4); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 4); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 4); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 4); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 4); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 4); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 4); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 0)) << 4; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 4); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 4); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 4); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 4); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 4); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 4); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 4); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 4); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 4); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 4); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 4); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 4); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 4); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 4); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 4); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<5>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 5); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 5); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 5); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 5); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 5); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 5); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 5); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 5); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 5); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 5); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 5); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 5); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 1)) << 4; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 5); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 5); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 5); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 5); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 5); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 5); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 5); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 5); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 5); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 5); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 5); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 5); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 2)) << 3; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 5); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 5); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 5); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 5); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 5); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 5); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 5); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 5); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 5); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 5); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 5); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 5); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 3)) << 2; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 5); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 5); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 5); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 5); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 5); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 5); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 5); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 5); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 5); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 5); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 5); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 5); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 4)) << 1; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 5); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 5); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 5); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 5); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 5); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 5); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 5); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 5); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 5); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 5); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 5); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<6>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 6); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 6); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 6); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 6); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 6); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 6); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 6); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 6); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 6); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 6); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 2)) << 4; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 6); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 6); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 6); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 6); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 6); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 6); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 6); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 6); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 6); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 6); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 4)) << 2; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 6); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 6); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 6); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 6); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 6); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 6); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 6); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 6); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 6); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 0)) << 6; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 6); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 6); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 6); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 6); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 6); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 6); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 6); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 6); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 6); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 6); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 2)) << 4; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 6); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 6); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 6); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 6); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 6); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 6); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 6); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 6); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 6); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 6); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 4)) << 2; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 6); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 6); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 6); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 6); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 6); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 6); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 6); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 6); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 6); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<7>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 7); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 7); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 7); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 7); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 7); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 7); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 7); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 7); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 7); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 6)) << 1; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 7); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 7); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 7); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 7); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 7); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 7); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 7); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 7); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 5)) << 2; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 7); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 7); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 7); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 7); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 7); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 7); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 7); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 7); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 4)) << 3; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 7); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 7); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 7); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 7); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 7); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 7); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 7); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 7); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 3)) << 4; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 7); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 7); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 7); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 7); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 7); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 7); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 7); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 7); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 2)) << 5; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 7); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 7); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 7); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 7); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 7); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 7); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 7); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 7); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 1)) << 6; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 7); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 7); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 7); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 7); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 7); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 7); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 7); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 7); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<8>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 8); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 8); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 8); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 8); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 8); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 8); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 8); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 0)) << 8; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 8); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 8); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 8); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 8); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 8); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 8); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 8); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 0)) << 8; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 8); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 8); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 8); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 8); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 8); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 8); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 8); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 0)) << 8; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 8); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 8); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 8); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 8); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 8); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 8); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 8); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 0)) << 8; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 8); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 8); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 8); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 8); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 8); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 8); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 8); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 0)) << 8; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 8); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 8); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 8); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 8); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 8); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 8); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 8); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 0)) << 8; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 8); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 8); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 8); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 8); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 8); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 8); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 8); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 0)) << 8; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 8); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 8); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 8); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 8); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 8); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 8); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 8); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<9>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 9); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 9); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 9); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 9); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 9); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 9); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 9); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 8)) << 1; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 9); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 9); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 9); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 9); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 9); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 9); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 7)) << 2; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 9); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 9); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 9); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 9); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 9); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 9); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 6)) << 3; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 9); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 9); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 9); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 9); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 9); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 9); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 5)) << 4; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 9); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 9); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 9); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 9); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 9); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 9); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 4)) << 5; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 9); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 9); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 9); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 9); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 9); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 9); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 3)) << 6; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 9); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 9); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 9); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 9); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 9); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 9); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 2)) << 7; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 9); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 9); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 9); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 9); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 9); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 9); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 1)) << 8; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 9); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 9); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 9); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 9); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 9); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 9); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<10>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 10); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 10); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 10); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 10); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 10); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 10); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 6)) << 4; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 10); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 10); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 10); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 10); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 10); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 2)) << 8; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 10); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 10); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 10); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 10); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 10); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 10); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 8)) << 2; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 10); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 10); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 10); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 10); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 10); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 4)) << 6; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 10); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 10); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 10); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 10); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 10); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 0)) << 10; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 10); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 10); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 10); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 10); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 10); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 10); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 6)) << 4; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 10); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 10); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 10); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 10); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 10); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 2)) << 8; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 10); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 10); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 10); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 10); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 10); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 10); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 8)) << 2; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 10); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 10); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 10); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 10); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 10); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 4)) << 6; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 10); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 10); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 10); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 10); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 10); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<11>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 11); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 11); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 11); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 11); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 11); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 2)) << 9; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 11); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 11); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 11); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 11); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 11); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 4)) << 7; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 11); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 11); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 11); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 11); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 11); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 6)) << 5; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 11); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 11); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 11); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 11); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 11); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 8)) << 3; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 11); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 11); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 11); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 11); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 11); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 10)) << 1; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 11); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 11); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 11); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 11); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 1)) << 10; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 11); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 11); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 11); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 11); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 11); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 3)) << 8; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 11); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 11); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 11); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 11); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 11); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 5)) << 6; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 11); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 11); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 11); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 11); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 11); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 7)) << 4; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 11); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 11); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 11); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 11); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 11); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 9)) << 2; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 11); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 11); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 11); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 11); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<12>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 12); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 12); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 12); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 12); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 12); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 8)) << 4; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 12); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 12); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 12); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 12); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 4)) << 8; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 12); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 12); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 12); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 12); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 0)) << 12; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 12); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 12); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 12); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 12); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 12); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 8)) << 4; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 12); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 12); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 12); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 12); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 4)) << 8; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 12); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 12); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 12); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 12); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 0)) << 12; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 12); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 12); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 12); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 12); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 12); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 8)) << 4; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 12); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 12); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 12); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 12); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 4)) << 8; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 12); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 12); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 12); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 12); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 0)) << 12; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 12); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 12); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 12); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 12); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 12); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 8)) << 4; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 12); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 12); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 12); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 12); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 4)) << 8; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 12); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 12); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 12); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 12); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<13>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 13); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 13); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 13); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 13); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 1)) << 12; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 13); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 13); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 13); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 13); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 2)) << 11; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 13); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 13); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 13); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 13); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 3)) << 10; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 13); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 13); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 13); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 13); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 4)) << 9; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 13); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 13); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 13); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 13); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 5)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 13); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 13); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 13); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 13); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 6)) << 7; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 13); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 13); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 13); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 13); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 7)) << 6; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 13); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 13); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 13); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 13); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 8)) << 5; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 13); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 13); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 13); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 13); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 9)) << 4; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 13); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 13); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 13); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 13); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 10)) << 3; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 13); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 13); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 13); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 13); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 11)) << 2; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 13); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 13); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 13); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 13); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 12)) << 1; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 13); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 13); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 13); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<14>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 14); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 14); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 14); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 14); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 6)) << 8; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 14); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 14); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 14); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 14); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 12)) << 2; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 14); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 14); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 14); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 4)) << 10; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 14); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 14); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 14); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 14); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 10)) << 4; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 14); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 14); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 14); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 2)) << 12; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 14); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 14); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 14); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 14); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 8)) << 6; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 14); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 14); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 14); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 0)) << 14; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 14); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 14); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 14); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 14); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 6)) << 8; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 14); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 14); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 14); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 14); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 12)) << 2; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 14); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 14); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 14); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 4)) << 10; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 14); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 14); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 14); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 14); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 10)) << 4; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 14); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 14); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 14); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 2)) << 12; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 14); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 14); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 14); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 14); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 6; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 14); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 14); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 14); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<15>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 15); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 15); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 15); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 15); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 11)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 15); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 15); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 15); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 7)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 15); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 15); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 15); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 3)) << 12; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 15); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 15); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 15); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 15); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 14)) << 1; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 15); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 15); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 15); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 10)) << 5; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 15); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 15); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 15); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 6)) << 9; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 15); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 15); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 15); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 2)) << 13; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 15); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 15); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 15); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 15); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 13)) << 2; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 15); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 15); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 15); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 9)) << 6; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 15); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 15); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 15); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 5)) << 10; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 15); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 15); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 15); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 1)) << 14; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 15); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 15); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 15); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 15); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 12)) << 3; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 15); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 15); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 15); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 7; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 15); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 15); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 15); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 4)) << 11; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 15); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 15); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 15); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<16>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<17>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 17); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 17); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 17); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 4)) << 13; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 17); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 17); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 17); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 8)) << 9; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 17); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 17); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 17); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 12)) << 5; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 17); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 17); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 17); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 16)) << 1; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 17); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 17); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 3)) << 14; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 17); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 17); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 17); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 7)) << 10; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 17); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 17); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 17); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 11)) << 6; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 17); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 17); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 17); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 15)) << 2; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 17); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 17); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 2)) << 15; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 17); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 17); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 17); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 6)) << 11; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 17); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 17); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 17); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 10)) << 7; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 17); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 17); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 17); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 14)) << 3; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 17); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 17); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 1)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 17); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 17); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 17); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 5)) << 12; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 17); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 17); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 17); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 9)) << 8; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 17); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 17); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 17); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 13)) << 4; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 17); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 17); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<18>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 18); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 18); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 18); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 8)) << 10; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 18); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 18); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 18); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 16)) << 2; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 18); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 18); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 6)) << 12; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 18); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 18); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 18); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 14)) << 4; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 18); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 18); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 4)) << 14; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 18); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 18); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 18); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 12)) << 6; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 18); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 18); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 2)) << 16; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 18); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 18); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 18); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 10)) << 8; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 18); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 18); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 0)) << 18; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 18); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 18); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 18); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 8)) << 10; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 18); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 18); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 18); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 16)) << 2; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 18); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 18); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 6)) << 12; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 18); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 18); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 18); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 14)) << 4; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 18); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 18); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 4)) << 14; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 18); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 18); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 18); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 12)) << 6; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 18); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 18); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 2)) << 16; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 18); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 18); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 18); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 10)) << 8; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 18); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 18); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<19>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 19); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 19); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 19); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 12)) << 7; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 19); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 19); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 5)) << 14; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 19); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 19); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 19); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 17)) << 2; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 19); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 19); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 10)) << 9; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 19); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 19); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 3)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 19); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 19); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 19); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 15)) << 4; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 19); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 19); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 8)) << 11; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 19); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 19); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 1)) << 18; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 19); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 19); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 19); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 13)) << 6; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 19); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 19); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 6)) << 13; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 19); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 19); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 19); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 18)) << 1; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 19); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 19); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 11)) << 8; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 19); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 19); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 4)) << 15; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 19); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 19); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 19); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 16)) << 3; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 19); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 19); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 9)) << 10; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 19); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 19); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 2)) << 17; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 19); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 19); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 19); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 14)) << 5; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 19); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 19); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 7)) << 12; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 19); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 19); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<20>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 20); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 20); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 20); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 16)) << 4; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 20); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 20); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 12)) << 8; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 20); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 20); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 8)) << 12; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 20); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 20); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 4)) << 16; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 20); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 20); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 0)) << 20; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 20); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 20); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 20); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 16)) << 4; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 20); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 20); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 12)) << 8; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 20); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 20); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 8)) << 12; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 20); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 20); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 4)) << 16; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 20); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 20); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 0)) << 20; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 20); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 20); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 20); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 16)) << 4; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 20); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 20); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 12)) << 8; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 20); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 20); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 12; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 20); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 20); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 4)) << 16; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 20); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 20); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 0)) << 20; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 20); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 20); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 20); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 16)) << 4; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 20); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 20); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 12)) << 8; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 20); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 20); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 8)) << 12; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 20); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 20); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 4)) << 16; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 20); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 20); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<21>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 21); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 21); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 21); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 20)) << 1; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 21); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 21); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 19)) << 2; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 21); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 21); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 18)) << 3; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 21); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 21); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 17)) << 4; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 21); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 21); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 16)) << 5; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 21); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 21); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 15)) << 6; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 21); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 21); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 14)) << 7; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 21); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 21); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 13)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 21); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 21); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 12)) << 9; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 21); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 21); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 11)) << 10; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 21); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 21); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 10)) << 11; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 21); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 21); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 9)) << 12; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 21); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 21); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 13; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 21); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 21); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 7)) << 14; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 21); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 21); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 6)) << 15; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 21); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 21); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 5)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 21); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 21); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 4)) << 17; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 21); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 21); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 3)) << 18; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 21); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 21); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 2)) << 19; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 21); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 21); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 1)) << 20; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 21); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 21); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<22>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 22); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 22); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 2)) << 20; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 22); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 22); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 4)) << 18; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 22); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 22); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 6)) << 16; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 22); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 22); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 8)) << 14; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 22); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 22); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 10)) << 12; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 22); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 22); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 12)) << 10; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 22); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 22); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 14)) << 8; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 22); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 22); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 16)) << 6; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 22); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 22); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 18)) << 4; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 22); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 22); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 20)) << 2; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 22); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 0)) << 22; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 22); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 22); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 2)) << 20; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 22); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 22); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 4)) << 18; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 22); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 22); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 6)) << 16; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 22); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 22); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 8)) << 14; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 22); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 22); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 10)) << 12; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 22); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 22); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 12)) << 10; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 22); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 22); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 14)) << 8; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 22); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 22); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 16)) << 6; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 22); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 22); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 18)) << 4; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 22); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 22); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 20)) << 2; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 22); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<23>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 23); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 23); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 5)) << 18; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 23); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 23); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 10)) << 13; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 23); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 23); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 15)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 23); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 23); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 20)) << 3; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 23); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 2)) << 21; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 23); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 23); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 7)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 23); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 23); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 12)) << 11; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 23); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 23); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 17)) << 6; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 23); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 23); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 22)) << 1; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 23); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 4)) << 19; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 23); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 23); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 9)) << 14; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 23); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 23); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 14)) << 9; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 23); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 23); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 19)) << 4; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 23); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 1)) << 22; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 23); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 23); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 6)) << 17; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 23); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 23); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 11)) << 12; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 23); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 23); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 16)) << 7; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 23); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 23); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 21)) << 2; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 23); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 3)) << 20; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 23); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 23); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 8)) << 15; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 23); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 23); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 13)) << 10; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 23); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 23); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 18)) << 5; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 23); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<24>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 24); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 24); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 8)) << 16; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 24); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 24); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 16)) << 8; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 24); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 0)) << 24; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 24); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 24); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 8)) << 16; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 24); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 24); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 16)) << 8; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 24); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 0)) << 24; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 24); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 24); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 8)) << 16; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 24); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 24); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 16)) << 8; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 24); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 0)) << 24; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 24); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 24); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 8)) << 16; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 24); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 24); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 16)) << 8; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 24); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 0)) << 24; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 24); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 24); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 16; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 24); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 24); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 16)) << 8; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 24); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 0)) << 24; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 24); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 24); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 8)) << 16; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 24); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 24); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 16)) << 8; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 24); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 0)) << 24; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 24); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 24); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 8)) << 16; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 24); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 24); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 16)) << 8; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 24); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 0)) << 24; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 24); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 24); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 8)) << 16; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 24); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 24); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 16)) << 8; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 24); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<25>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 25); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 25); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 11)) << 14; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 25); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 25); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 22)) << 3; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 25); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 8)) << 17; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 25); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 25); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 19)) << 6; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 25); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 5)) << 20; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 25); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 25); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 16)) << 9; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 25); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 2)) << 23; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 25); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 25); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 13)) << 12; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 25); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 25); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 24)) << 1; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 25); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 10)) << 15; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 25); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 25); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 21)) << 4; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 25); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 7)) << 18; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 25); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 25); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 18)) << 7; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 25); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 4)) << 21; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 25); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 25); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 15)) << 10; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 25); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 1)) << 24; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 25); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 25); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 12)) << 13; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 25); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 25); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 23)) << 2; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 25); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 9)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 25); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 25); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 20)) << 5; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 25); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 6)) << 19; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 25); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 25); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 17)) << 8; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 25); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 3)) << 22; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 25); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 25); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 14)) << 11; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 25); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<26>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 26); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 26); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 14)) << 12; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 26); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 2)) << 24; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 26); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 26); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 16)) << 10; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 26); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 4)) << 22; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 26); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 26); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 18)) << 8; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 26); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 6)) << 20; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 26); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 26); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 20)) << 6; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 26); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 8)) << 18; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 26); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 26); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 22)) << 4; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 26); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 10)) << 16; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 26); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 26); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 24)) << 2; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 26); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 12)) << 14; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 26); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 0)) << 26; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 26); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 26); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 14)) << 12; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 26); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 2)) << 24; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 26); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 26); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 16)) << 10; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 26); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 4)) << 22; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 26); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 26); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 18)) << 8; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 26); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 6)) << 20; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 26); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 26); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 20)) << 6; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 26); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 8)) << 18; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 26); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 26); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 22)) << 4; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 26); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 10)) << 16; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 26); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 26); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 24)) << 2; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 26); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 12)) << 14; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 26); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<27>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 27); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 27); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 17)) << 10; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 27); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 7)) << 20; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 27); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 27); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 24)) << 3; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 27); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 14)) << 13; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 27); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 4)) << 23; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 27); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 27); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 21)) << 6; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 27); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 11)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 27); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 1)) << 26; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 27); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 27); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 18)) << 9; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 27); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 8)) << 19; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 27); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 27); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 25)) << 2; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 27); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 15)) << 12; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 27); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 5)) << 22; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 27); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 27); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 22)) << 5; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 27); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 12)) << 15; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 27); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 2)) << 25; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 27); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 27); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 19)) << 8; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 27); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 9)) << 18; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 27); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 27); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 26)) << 1; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 27); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 16)) << 11; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 27); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 6)) << 21; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 27); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 27); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 23)) << 4; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 27); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 13)) << 14; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 27); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 3)) << 24; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 27); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 27); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 20)) << 7; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 27); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 10)) << 17; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 27); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<28>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 28); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 28); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 20)) << 8; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 28); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 12)) << 16; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 28); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 4)) << 24; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 28); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 28); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 24)) << 4; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 28); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 16)) << 12; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 28); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 8)) << 20; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 28); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 0)) << 28; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 28); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 28); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 20)) << 8; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 28); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 12)) << 16; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 28); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 4)) << 24; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 28); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 28); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 24)) << 4; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 28); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 16)) << 12; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 28); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 20; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 28); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 0)) << 28; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 28); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 28); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 20)) << 8; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 28); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 12)) << 16; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 28); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 4)) << 24; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 28); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 28); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 24)) << 4; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 28); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 16)) << 12; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 28); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 8)) << 20; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 28); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 0)) << 28; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 28); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 28); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 20)) << 8; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 28); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 12)) << 16; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 28); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 4)) << 24; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 28); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 28); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 24)) << 4; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 28); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 16)) << 12; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 28); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 8)) << 20; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 28); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<29>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 29); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 29); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 23)) << 6; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 29); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 17)) << 12; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 29); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 11)) << 18; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 29); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 5)) << 24; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 29); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 29); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 28)) << 1; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 29); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 22)) << 7; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 29); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 16)) << 13; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 29); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 10)) << 19; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 29); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 4)) << 25; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 29); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 29); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 27)) << 2; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 29); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 21)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 29); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 15)) << 14; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 29); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 9)) << 20; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 29); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 3)) << 26; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 29); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 29); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 26)) << 3; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 29); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 20)) << 9; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 29); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 14)) << 15; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 29); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 8)) << 21; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 29); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 2)) << 27; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 29); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 29); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 25)) << 4; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 29); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 19)) << 10; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 29); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 13)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 29); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 7)) << 22; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 29); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 1)) << 28; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 29); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 29); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 24)) << 5; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 29); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 18)) << 11; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 29); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 12)) << 17; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 29); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 6)) << 23; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 29); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<30>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 30); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 30); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 26)) << 4; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 30); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 22)) << 8; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 30); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 18)) << 12; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 30); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 14)) << 16; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 30); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 10)) << 20; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 30); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 6)) << 24; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 30); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 2)) << 28; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 30); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 30); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 28)) << 2; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 30); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 24)) << 6; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 30); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 20)) << 10; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 30); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 16)) << 14; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 30); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 12)) << 18; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 30); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 22; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 30); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 4)) << 26; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 30); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 0)) << 30; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 30); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 30); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 26)) << 4; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 30); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 22)) << 8; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 30); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 18)) << 12; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 30); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 14)) << 16; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 30); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 10)) << 20; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 30); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 6)) << 24; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 30); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 2)) << 28; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 30); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 30); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 28)) << 2; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 30); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 24)) << 6; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 30); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 20)) << 10; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 30); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 16)) << 14; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 30); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 12)) << 18; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 30); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 8)) << 22; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 30); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 4)) << 26; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 30); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<31>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 31); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 31); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 29)) << 2; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 31); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 27)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 31); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 25)) << 6; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 31); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 23)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 31); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 21)) << 10; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 31); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 19)) << 12; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 31); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 17)) << 14; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 31); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 15)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 31); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 13)) << 18; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 31); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 11)) << 20; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 31); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 9)) << 22; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 31); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 7)) << 24; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 31); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 5)) << 26; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 31); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 3)) << 28; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 31); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 1)) << 30; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 31); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 31); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 30)) << 1; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 31); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 28)) << 3; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 31); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 26)) << 5; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 31); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 24)) << 7; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 31); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 22)) << 9; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 31); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 20)) << 11; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 31); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 18)) << 13; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 31); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 16)) << 15; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 31); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 14)) << 17; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 31); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 12)) << 19; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 31); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 10)) << 21; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 31); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 8)) << 23; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 31); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 6)) << 25; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 31); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 4)) << 27; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 31); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 2)) << 29; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 31); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<32>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<33>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 33); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 2)) << 31; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 33); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 4)) << 29; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 33); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 6)) << 27; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 33); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 8)) << 25; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 33); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 10)) << 23; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 33); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 12)) << 21; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 33); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 14)) << 19; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 33); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 16)) << 17; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 33); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 18)) << 15; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 33); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 20)) << 13; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 33); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 22)) << 11; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 33); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 24)) << 9; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 33); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 26)) << 7; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 33); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 28)) << 5; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 33); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 30)) << 3; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 33); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 32)) << 1; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 1)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 33); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 3)) << 30; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 33); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 5)) << 28; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 33); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 7)) << 26; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 33); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 9)) << 24; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 33); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 11)) << 22; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 33); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 13)) << 20; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 33); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 15)) << 18; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 33); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 17)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 33); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 19)) << 14; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 33); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 21)) << 12; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 33); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 23)) << 10; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 33); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 25)) << 8; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 33); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 27)) << 6; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 33); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 29)) << 4; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 33); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 31)) << 2; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<34>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 34); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 4)) << 30; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 34); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 8)) << 26; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 34); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 12)) << 22; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 34); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 16)) << 18; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 34); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 20)) << 14; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 34); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 24)) << 10; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 34); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 28)) << 6; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 34); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 32)) << 2; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 2)) << 32; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 34); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 6)) << 28; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 34); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 10)) << 24; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 34); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 14)) << 20; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 34); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 18)) << 16; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 34); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 22)) << 12; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 34); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 26)) << 8; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 34); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 30)) << 4; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 0)) << 34; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 34); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 4)) << 30; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 34); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 8)) << 26; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 34); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 12)) << 22; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 34); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 16)) << 18; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 34); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 20)) << 14; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 34); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 24)) << 10; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 34); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 28)) << 6; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 34); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 32)) << 2; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 2)) << 32; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 34); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 6)) << 28; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 34); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 10)) << 24; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 34); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 14)) << 20; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 34); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 18)) << 16; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 34); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 22)) << 12; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 34); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 26)) << 8; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 34); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 30)) << 4; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<35>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 35); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 6)) << 29; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 35); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 12)) << 23; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 35); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 18)) << 17; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 35); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 24)) << 11; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 35); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 30)) << 5; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 1)) << 34; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 35); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 7)) << 28; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 35); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 13)) << 22; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 35); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 19)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 35); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 25)) << 10; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 35); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 31)) << 4; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 2)) << 33; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 35); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 27; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 35); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 14)) << 21; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 35); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 20)) << 15; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 35); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 26)) << 9; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 35); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 32)) << 3; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 3)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 35); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 9)) << 26; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 35); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 15)) << 20; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 35); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 21)) << 14; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 35); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 27)) << 8; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 35); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 33)) << 2; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 4)) << 31; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 35); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 10)) << 25; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 35); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 16)) << 19; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 35); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 22)) << 13; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 35); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 28)) << 7; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 35); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 34)) << 1; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 5)) << 30; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 35); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 11)) << 24; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 35); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 17)) << 18; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 35); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 23)) << 12; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 35); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 29)) << 6; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<36>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 36); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 8)) << 28; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 36); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 16)) << 20; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 36); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 24)) << 12; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 36); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 32)) << 4; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 4)) << 32; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 36); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 12)) << 24; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 36); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 20)) << 16; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 36); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 28)) << 8; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 0)) << 36; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 36); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 8)) << 28; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 36); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 16)) << 20; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 36); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 24)) << 12; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 36); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 32)) << 4; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 4)) << 32; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 36); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 12)) << 24; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 36); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 20)) << 16; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 36); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 28)) << 8; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 0)) << 36; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 36); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 8)) << 28; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 36); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 16)) << 20; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 36); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 24)) << 12; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 36); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 32)) << 4; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 4)) << 32; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 36); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 12)) << 24; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 36); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 20)) << 16; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 36); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 28)) << 8; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 0)) << 36; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 36); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 8)) << 28; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 36); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 16)) << 20; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 36); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 24)) << 12; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 36); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 32)) << 4; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 4)) << 32; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 36); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 12)) << 24; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 36); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 20)) << 16; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 36); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 28)) << 8; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<37>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 37); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 10)) << 27; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 37); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 20)) << 17; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 37); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 30)) << 7; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 3)) << 34; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 37); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 13)) << 24; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 37); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 23)) << 14; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 37); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 33)) << 4; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 6)) << 31; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 37); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 16)) << 21; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 37); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 26)) << 11; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 37); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 36)) << 1; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 9)) << 28; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 37); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 19)) << 18; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 37); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 29)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 2)) << 35; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 37); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 12)) << 25; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 37); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 22)) << 15; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 37); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 32)) << 5; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 5)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 37); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 15)) << 22; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 37); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 25)) << 12; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 37); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 35)) << 2; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 8)) << 29; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 37); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 18)) << 19; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 37); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 28)) << 9; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 1)) << 36; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 37); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 11)) << 26; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 37); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 21)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 37); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 31)) << 6; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 4)) << 33; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 37); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 14)) << 23; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 37); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 24)) << 13; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 37); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 34)) << 3; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 7)) << 30; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 37); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 17)) << 20; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 37); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 27)) << 10; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<38>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 38); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 12)) << 26; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 38); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 24)) << 14; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 38); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 36)) << 2; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 10)) << 28; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 38); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 22)) << 16; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 38); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 34)) << 4; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 8)) << 30; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 38); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 20)) << 18; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 38); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 32)) << 6; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 6)) << 32; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 38); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 18)) << 20; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 38); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 30)) << 8; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 4)) << 34; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 38); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 16)) << 22; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 38); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 28)) << 10; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 2)) << 36; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 38); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 14)) << 24; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 38); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 26)) << 12; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 0)) << 38; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 38); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 12)) << 26; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 38); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 24)) << 14; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 38); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 36)) << 2; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 10)) << 28; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 38); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 22)) << 16; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 38); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 34)) << 4; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 8)) << 30; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 38); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 20)) << 18; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 38); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 32)) << 6; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 6)) << 32; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 38); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 18)) << 20; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 38); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 30)) << 8; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 4)) << 34; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 38); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 16)) << 22; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 38); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 28)) << 10; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 2)) << 36; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 38); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 14)) << 24; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 38); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 26)) << 12; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<39>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 39); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 14)) << 25; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 39); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 28)) << 11; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 3)) << 36; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 39); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 17)) << 22; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 39); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 31)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 6)) << 33; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 39); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 20)) << 19; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 39); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 34)) << 5; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 9)) << 30; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 39); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 23)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 39); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 37)) << 2; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 12)) << 27; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 39); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 26)) << 13; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 1)) << 38; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 39); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 15)) << 24; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 39); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 29)) << 10; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 4)) << 35; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 39); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 18)) << 21; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 39); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 32)) << 7; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 7)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 39); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 21)) << 18; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 39); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 35)) << 4; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 10)) << 29; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 39); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 24)) << 15; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 39); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 38)) << 1; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 13)) << 26; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 39); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 27)) << 12; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 2)) << 37; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 39); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 16)) << 23; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 39); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 30)) << 9; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 5)) << 34; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 39); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 19)) << 20; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 39); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 33)) << 6; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 8)) << 31; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 39); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 22)) << 17; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 39); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 36)) << 3; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 11)) << 28; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 39); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 25)) << 14; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<40>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 40); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 16)) << 24; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 40); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 32)) << 8; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 8)) << 32; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 40); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 24)) << 16; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 0)) << 40; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 40); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 16)) << 24; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 40); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 32)) << 8; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 8)) << 32; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 40); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 24)) << 16; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 0)) << 40; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 40); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 16)) << 24; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 40); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 32)) << 8; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 32; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 40); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 24)) << 16; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 0)) << 40; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 40); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 16)) << 24; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 40); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 32)) << 8; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 8)) << 32; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 40); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 24)) << 16; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 0)) << 40; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 40); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 16)) << 24; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 40); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 32)) << 8; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 8)) << 32; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 40); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 24)) << 16; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 0)) << 40; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 40); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 16)) << 24; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 40); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 32)) << 8; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 8)) << 32; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 40); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 24)) << 16; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 0)) << 40; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 40); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 16)) << 24; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 40); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 32)) << 8; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 8)) << 32; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 40); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 24)) << 16; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 0)) << 40; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 40); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 16)) << 24; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 40); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 32)) << 8; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 8)) << 32; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 40); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 24)) << 16; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<41>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 41); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 18)) << 23; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 41); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 36)) << 5; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 13)) << 28; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 41); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 31)) << 10; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 8)) << 33; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 41); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 26)) << 15; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 3)) << 38; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 41); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 21)) << 20; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 41); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 39)) << 2; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 16)) << 25; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 41); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 34)) << 7; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 11)) << 30; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 41); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 29)) << 12; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 6)) << 35; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 41); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 24)) << 17; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 1)) << 40; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 41); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 19)) << 22; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 41); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 37)) << 4; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 14)) << 27; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 41); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 32)) << 9; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 9)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 41); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 27)) << 14; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 4)) << 37; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 41); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 22)) << 19; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 41); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 40)) << 1; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 17)) << 24; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 41); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 35)) << 6; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 12)) << 29; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 41); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 30)) << 11; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 7)) << 34; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 41); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 25)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 2)) << 39; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 41); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 20)) << 21; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 41); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 38)) << 3; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 15)) << 26; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 41); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 33)) << 8; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 10)) << 31; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 41); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 28)) << 13; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 5)) << 36; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 41); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 23)) << 18; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<42>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 42); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 20)) << 22; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 42); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 40)) << 2; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 18)) << 24; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 42); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 38)) << 4; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 16)) << 26; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 42); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 36)) << 6; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 14)) << 28; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 42); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 34)) << 8; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 12)) << 30; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 42); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 32)) << 10; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 10)) << 32; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 42); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 30)) << 12; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 34; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 42); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 28)) << 14; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 6)) << 36; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 42); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 26)) << 16; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 4)) << 38; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 42); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 24)) << 18; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 2)) << 40; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 42); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 22)) << 20; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 0)) << 42; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 42); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 20)) << 22; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 42); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 40)) << 2; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 18)) << 24; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 42); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 38)) << 4; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 16)) << 26; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 42); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 36)) << 6; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 14)) << 28; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 42); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 34)) << 8; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 12)) << 30; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 42); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 32)) << 10; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 10)) << 32; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 42); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 30)) << 12; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 8)) << 34; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 42); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 28)) << 14; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 6)) << 36; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 42); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 26)) << 16; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 4)) << 38; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 42); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 24)) << 18; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 2)) << 40; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 42); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 22)) << 20; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<43>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 43); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 22)) << 21; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 1)) << 42; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 43); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 23)) << 20; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 2)) << 41; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 43); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 24)) << 19; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 3)) << 40; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 43); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 25)) << 18; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 4)) << 39; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 43); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 26)) << 17; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 5)) << 38; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 43); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 27)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 6)) << 37; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 43); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 28)) << 15; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 7)) << 36; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 43); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 29)) << 14; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 8)) << 35; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 43); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 30)) << 13; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 9)) << 34; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 43); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 31)) << 12; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 10)) << 33; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 43); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 32)) << 11; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 11)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 43); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 33)) << 10; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 12)) << 31; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 43); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 34)) << 9; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 13)) << 30; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 43); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 35)) << 8; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 14)) << 29; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 43); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 36)) << 7; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 15)) << 28; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 43); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 37)) << 6; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 16)) << 27; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 43); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 38)) << 5; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 17)) << 26; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 43); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 39)) << 4; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 18)) << 25; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 43); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 40)) << 3; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 19)) << 24; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 43); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 41)) << 2; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 20)) << 23; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 43); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 42)) << 1; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 21)) << 22; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 43); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<44>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 44); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 24)) << 20; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 4)) << 40; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 44); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 28)) << 16; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 8)) << 36; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 44); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 32)) << 12; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 12)) << 32; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 44); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 36)) << 8; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 16)) << 28; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 44); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 40)) << 4; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 20)) << 24; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 0)) << 44; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 44); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 24)) << 20; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 4)) << 40; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 44); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 28)) << 16; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 8)) << 36; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 44); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 32)) << 12; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 12)) << 32; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 44); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 36)) << 8; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 16)) << 28; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 44); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 40)) << 4; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 20)) << 24; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 0)) << 44; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 44); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 24)) << 20; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 4)) << 40; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 44); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 28)) << 16; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 8)) << 36; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 44); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 32)) << 12; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 12)) << 32; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 44); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 36)) << 8; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 16)) << 28; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 44); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 40)) << 4; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 20)) << 24; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 0)) << 44; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 44); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 24)) << 20; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 4)) << 40; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 44); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 28)) << 16; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 8)) << 36; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 44); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 32)) << 12; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 12)) << 32; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 44); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 36)) << 8; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 16)) << 28; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 44); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 40)) << 4; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 20)) << 24; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<45>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 45); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 26)) << 19; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 7)) << 38; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 45); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 33)) << 12; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 14)) << 31; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 45); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 40)) << 5; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 21)) << 24; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 43); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 2)) << 43; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 45); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 28)) << 17; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 9)) << 36; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 45); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 35)) << 10; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 16)) << 29; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 45); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 42)) << 3; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 23)) << 22; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 4)) << 41; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 45); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 30)) << 15; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 11)) << 34; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 45); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 37)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 18)) << 27; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 45); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 44)) << 1; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 25)) << 20; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 6)) << 39; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 45); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 32)) << 13; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 13)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 45); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 39)) << 6; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 20)) << 25; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 1)) << 44; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 45); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 27)) << 18; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 8)) << 37; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 45); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 34)) << 11; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 15)) << 30; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 45); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 41)) << 4; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 22)) << 23; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 3)) << 42; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 45); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 29)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 10)) << 35; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 45); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 36)) << 9; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 17)) << 28; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 45); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 43)) << 2; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 24)) << 21; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 5)) << 40; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 45); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 31)) << 14; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 12)) << 33; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 45); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 38)) << 7; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 19)) << 26; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 45); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<46>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 46); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 28)) << 18; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 10)) << 36; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 46); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 38)) << 8; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 20)) << 26; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 2)) << 44; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 46); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 30)) << 16; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 12)) << 34; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 46); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 40)) << 6; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 22)) << 24; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 4)) << 42; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 46); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 32)) << 14; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 14)) << 32; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 46); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 42)) << 4; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 24)) << 22; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 6)) << 40; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 46); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 34)) << 12; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 16)) << 30; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 46); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 44)) << 2; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 26)) << 20; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 8)) << 38; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 46); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 36)) << 10; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 18)) << 28; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 0)) << 46; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 46); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 28)) << 18; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 10)) << 36; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 46); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 38)) << 8; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 20)) << 26; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 2)) << 44; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 46); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 30)) << 16; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 12)) << 34; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 46); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 40)) << 6; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 22)) << 24; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 4)) << 42; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 46); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 32)) << 14; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 14)) << 32; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 46); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 42)) << 4; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 24)) << 22; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 6)) << 40; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 46); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 34)) << 12; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 16)) << 30; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 46); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 44)) << 2; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 26)) << 20; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 8)) << 38; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 46); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 36)) << 10; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 18)) << 28; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<47>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 47); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 30)) << 17; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 13)) << 34; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 47); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 43)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 26)) << 21; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 9)) << 38; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 47); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 39)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 22)) << 25; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 5)) << 42; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 47); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 35)) << 12; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 18)) << 29; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 1)) << 46; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 47); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 31)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 14)) << 33; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 47); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 44)) << 3; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 27)) << 20; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 10)) << 37; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 47); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 40)) << 7; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 23)) << 24; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 6)) << 41; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 47); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 36)) << 11; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 19)) << 28; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 45); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 2)) << 45; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 47); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 32)) << 15; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 15)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 47); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 45)) << 2; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 28)) << 19; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 11)) << 36; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 47); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 41)) << 6; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 24)) << 23; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 7)) << 40; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 47); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 37)) << 10; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 20)) << 27; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 3)) << 44; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 47); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 33)) << 14; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 16)) << 31; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 47); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 46)) << 1; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 29)) << 18; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 12)) << 35; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 47); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 42)) << 5; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 25)) << 22; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 8)) << 39; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 47); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 38)) << 9; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 21)) << 26; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 43); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 4)) << 43; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 47); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 34)) << 13; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 17)) << 30; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 47); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<48>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<49>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 49); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 34)) << 15; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 19)) << 30; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 45); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 4)) << 45; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 49); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 38)) << 11; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 23)) << 26; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 8)) << 41; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 49); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 42)) << 7; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 27)) << 22; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 12)) << 37; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 49); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 46)) << 3; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 31)) << 18; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 16)) << 33; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 1)) << 48; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 49); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 35)) << 14; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 20)) << 29; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 5)) << 44; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 49); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 39)) << 10; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 24)) << 25; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 9)) << 40; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 49); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 43)) << 6; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 28)) << 21; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 13)) << 36; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 49); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 47)) << 2; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 32)) << 17; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 17)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 47); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 2)) << 47; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 49); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 36)) << 13; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 21)) << 28; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 43); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 6)) << 43; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 49); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 40)) << 9; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 25)) << 24; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 10)) << 39; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 49); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 44)) << 5; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 29)) << 20; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 14)) << 35; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 49); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 48)) << 1; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 33)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 18)) << 31; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 3)) << 46; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 49); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 37)) << 12; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 22)) << 27; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 7)) << 42; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 49); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 41)) << 8; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 26)) << 23; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 11)) << 38; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 49); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 45)) << 4; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 30)) << 19; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 15)) << 34; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 49); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<50>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 50); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 36)) << 14; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 22)) << 28; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 8)) << 42; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 50); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 44)) << 6; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 30)) << 20; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 16)) << 34; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 2)) << 48; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 50); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 38)) << 12; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 24)) << 26; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 10)) << 40; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 50); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 46)) << 4; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 32)) << 18; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 18)) << 32; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 4)) << 46; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 50); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 40)) << 10; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 26)) << 24; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 12)) << 38; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 50); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 48)) << 2; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 34)) << 16; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 20)) << 30; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 6)) << 44; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 50); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 42)) << 8; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 28)) << 22; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 14)) << 36; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 0)) << 50; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 50); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 36)) << 14; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 22)) << 28; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 8)) << 42; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 50); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 44)) << 6; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 30)) << 20; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 16)) << 34; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 2)) << 48; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 50); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 38)) << 12; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 24)) << 26; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 10)) << 40; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 50); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 46)) << 4; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 32)) << 18; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 18)) << 32; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 4)) << 46; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 50); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 40)) << 10; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 26)) << 24; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 12)) << 38; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 50); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 48)) << 2; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 34)) << 16; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 20)) << 30; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 6)) << 44; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 50); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 42)) << 8; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 28)) << 22; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 14)) << 36; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<51>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 51); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 38)) << 13; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 25)) << 26; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 12)) << 39; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 51); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 50)) << 1; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 37)) << 14; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 24)) << 27; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 11)) << 40; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 51); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 49)) << 2; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 36)) << 15; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 23)) << 28; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 10)) << 41; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 51); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 48)) << 3; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 35)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 22)) << 29; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 9)) << 42; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 51); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 47)) << 4; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 34)) << 17; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 21)) << 30; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 43); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 8)) << 43; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 51); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 46)) << 5; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 33)) << 18; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 20)) << 31; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 7)) << 44; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 51); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 45)) << 6; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 32)) << 19; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 19)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 45); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 6)) << 45; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 51); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 44)) << 7; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 31)) << 20; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 18)) << 33; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 5)) << 46; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 51); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 43)) << 8; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 30)) << 21; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 17)) << 34; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 47); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 4)) << 47; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 51); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 42)) << 9; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 29)) << 22; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 16)) << 35; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 3)) << 48; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 51); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 41)) << 10; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 28)) << 23; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 15)) << 36; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 49); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 2)) << 49; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 51); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 40)) << 11; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 27)) << 24; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 14)) << 37; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 1)) << 50; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 51); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 39)) << 12; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 26)) << 25; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 13)) << 38; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 51); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<52>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 52); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 40)) << 12; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 28)) << 24; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 16)) << 36; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 4)) << 48; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 52); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 44)) << 8; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 32)) << 20; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 20)) << 32; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 8)) << 44; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 52); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 48)) << 4; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 36)) << 16; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 24)) << 28; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 12)) << 40; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 0)) << 52; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 52); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 40)) << 12; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 28)) << 24; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 16)) << 36; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 4)) << 48; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 52); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 44)) << 8; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 32)) << 20; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 20)) << 32; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 8)) << 44; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 52); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 48)) << 4; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 36)) << 16; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 24)) << 28; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 12)) << 40; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 0)) << 52; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 52); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 40)) << 12; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 28)) << 24; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 16)) << 36; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 4)) << 48; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 52); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 44)) << 8; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 32)) << 20; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 20)) << 32; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 8)) << 44; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 52); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 48)) << 4; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 36)) << 16; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 24)) << 28; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 12)) << 40; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 0)) << 52; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 52); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 40)) << 12; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 28)) << 24; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 16)) << 36; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 4)) << 48; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 52); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 44)) << 8; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 32)) << 20; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 20)) << 32; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 8)) << 44; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 52); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 48)) << 4; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 36)) << 16; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 24)) << 28; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 12)) << 40; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<53>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 53); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 42)) << 11; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 31)) << 22; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 20)) << 33; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 9)) << 44; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 53); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 51)) << 2; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 40)) << 13; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 29)) << 24; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 18)) << 35; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 7)) << 46; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 53); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 49)) << 4; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 38)) << 15; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 27)) << 26; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 16)) << 37; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 5)) << 48; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 53); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 47)) << 6; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 36)) << 17; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 25)) << 28; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 14)) << 39; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 3)) << 50; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 53); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 45)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 34)) << 19; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 23)) << 30; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 12)) << 41; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 1)) << 52; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 53); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 43)) << 10; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 32)) << 21; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 21)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 43); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 10)) << 43; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 53); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 52)) << 1; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 41)) << 12; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 30)) << 23; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 19)) << 34; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 45); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 8)) << 45; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 53); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 50)) << 3; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 39)) << 14; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 28)) << 25; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 17)) << 36; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 47); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 6)) << 47; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 53); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 48)) << 5; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 37)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 26)) << 27; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 15)) << 38; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 49); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 4)) << 49; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 53); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 46)) << 7; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 35)) << 18; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 24)) << 29; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 13)) << 40; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 51); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 2)) << 51; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 53); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 44)) << 9; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 33)) << 20; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 22)) << 31; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 52]; - tmp |= (src & MASK(uint64_t, 11)) << 42; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 53); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<54>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 54); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 44)) << 10; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 34)) << 20; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 24)) << 30; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 14)) << 40; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 4)) << 50; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 54); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 48)) << 6; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 38)) << 16; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 28)) << 26; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 18)) << 36; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 8)) << 46; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 54); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 52)) << 2; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 42)) << 12; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 32)) << 22; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 22)) << 32; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 12)) << 42; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 2)) << 52; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 54); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 46)) << 8; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 36)) << 18; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 26)) << 28; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 16)) << 38; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 6)) << 48; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 54); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 50)) << 4; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 40)) << 14; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 30)) << 24; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 20)) << 34; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 10)) << 44; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 54); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 0)) << 54; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 54); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 44)) << 10; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 34)) << 20; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 24)) << 30; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 14)) << 40; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 4)) << 50; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 54); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 48)) << 6; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 38)) << 16; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 28)) << 26; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 18)) << 36; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 8)) << 46; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 54); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 52)) << 2; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 42)) << 12; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 32)) << 22; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 22)) << 32; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 12)) << 42; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 2)) << 52; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 54); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 46)) << 8; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 36)) << 18; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 26)) << 28; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 16)) << 38; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 6)) << 48; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 54); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 50)) << 4; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 40)) << 14; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 30)) << 24; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 52]; - tmp |= (src & MASK(uint64_t, 20)) << 34; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 53]; - tmp |= (src & MASK(uint64_t, 10)) << 44; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 54); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<55>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 55); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 46)) << 9; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 37)) << 18; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 28)) << 27; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 19)) << 36; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 45); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 10)) << 45; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 54); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 1)) << 54; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 55); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 47)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 38)) << 17; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 29)) << 26; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 20)) << 35; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 11)) << 44; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 53); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 2)) << 53; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 55); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 48)) << 7; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 39)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 30)) << 25; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 21)) << 34; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 43); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 12)) << 43; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 3)) << 52; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 55); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 49)) << 6; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 40)) << 15; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 31)) << 24; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 22)) << 33; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 13)) << 42; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 51); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 4)) << 51; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 55); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 50)) << 5; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 41)) << 14; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 32)) << 23; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 23)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 14)) << 41; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 5)) << 50; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 55); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 51)) << 4; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 42)) << 13; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 33)) << 22; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 24)) << 31; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 15)) << 40; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 49); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 6)) << 49; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 55); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 52)) << 3; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 43)) << 12; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 34)) << 21; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 25)) << 30; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 16)) << 39; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 7)) << 48; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 55); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 53)) << 2; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 44)) << 11; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 35)) << 20; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 26)) << 29; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 17)) << 38; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 47); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 8)) << 47; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 55); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 54)) << 1; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 45)) << 10; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 36)) << 19; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 52]; - tmp |= (src & MASK(uint64_t, 27)) << 28; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 53]; - tmp |= (src & MASK(uint64_t, 18)) << 37; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 54]; - tmp |= (src & MASK(uint64_t, 9)) << 46; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 55); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<56>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 56); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 48)) << 8; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 40)) << 16; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 32)) << 24; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 24)) << 32; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 16)) << 40; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 8)) << 48; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 0)) << 56; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 56); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 48)) << 8; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 40)) << 16; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 32)) << 24; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 24)) << 32; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 16)) << 40; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 48; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 0)) << 56; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 56); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 48)) << 8; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 40)) << 16; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 32)) << 24; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 24)) << 32; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 16)) << 40; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 8)) << 48; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 0)) << 56; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 56); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 48)) << 8; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 40)) << 16; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 32)) << 24; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 24)) << 32; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 16)) << 40; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 8)) << 48; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 0)) << 56; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 56); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 48)) << 8; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 40)) << 16; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 32)) << 24; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 24)) << 32; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 16)) << 40; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 8)) << 48; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 0)) << 56; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 56); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 48)) << 8; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 40)) << 16; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 32)) << 24; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 24)) << 32; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 16)) << 40; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 8)) << 48; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 0)) << 56; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 56); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 48)) << 8; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 40)) << 16; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 32)) << 24; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 24)) << 32; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 16)) << 40; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 8)) << 48; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 0)) << 56; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 56); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 48)) << 8; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 40)) << 16; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 52]; - tmp |= (src & MASK(uint64_t, 32)) << 24; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 53]; - tmp |= (src & MASK(uint64_t, 24)) << 32; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 54]; - tmp |= (src & MASK(uint64_t, 16)) << 40; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 55]; - tmp |= (src & MASK(uint64_t, 8)) << 48; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<57>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 57); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 50)) << 7; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 43)) << 14; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 36)) << 21; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 29)) << 28; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 22)) << 35; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 15)) << 42; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 49); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 8)) << 49; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 1)) << 56; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 57); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 51)) << 6; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 44)) << 13; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 37)) << 20; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 30)) << 27; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 23)) << 34; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 16)) << 41; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 9)) << 48; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 55); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 2)) << 55; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 57); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 52)) << 5; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 45)) << 12; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 38)) << 19; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 31)) << 26; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 24)) << 33; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 17)) << 40; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 47); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 10)) << 47; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 54); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 3)) << 54; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 57); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 53)) << 4; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 46)) << 11; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 39)) << 18; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 32)) << 25; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 25)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 18)) << 39; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 11)) << 46; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 53); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 4)) << 53; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 57); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 54)) << 3; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 47)) << 10; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 40)) << 17; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 33)) << 24; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 26)) << 31; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 19)) << 38; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 45); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 12)) << 45; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 5)) << 52; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 57); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 55)) << 2; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 48)) << 9; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 41)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 34)) << 23; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 27)) << 30; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 20)) << 37; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 13)) << 44; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 51); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 6)) << 51; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 57); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 56)) << 1; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 49)) << 8; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 42)) << 15; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 52]; - tmp |= (src & MASK(uint64_t, 35)) << 22; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 53]; - tmp |= (src & MASK(uint64_t, 28)) << 29; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 54]; - tmp |= (src & MASK(uint64_t, 21)) << 36; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 43); - src = in[lane + LANE_COUNT * 55]; - tmp |= (src & MASK(uint64_t, 14)) << 43; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 56]; - tmp |= (src & MASK(uint64_t, 7)) << 50; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 57); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<58>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 58); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 52)) << 6; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 46)) << 12; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 40)) << 18; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 34)) << 24; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 28)) << 30; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 22)) << 36; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 16)) << 42; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 10)) << 48; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 54); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 4)) << 54; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 58); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 56)) << 2; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 50)) << 8; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 44)) << 14; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 38)) << 20; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 32)) << 26; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 26)) << 32; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 20)) << 38; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 14)) << 44; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 8)) << 50; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 2)) << 56; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 58); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 54)) << 4; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 48)) << 10; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 42)) << 16; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 36)) << 22; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 30)) << 28; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 24)) << 34; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 18)) << 40; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 12)) << 46; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 6)) << 52; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 58); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 0)) << 58; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 58); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 52)) << 6; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 46)) << 12; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 40)) << 18; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 34)) << 24; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 28)) << 30; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 22)) << 36; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 16)) << 42; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 10)) << 48; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 54); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 4)) << 54; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 58); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 56)) << 2; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 50)) << 8; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 44)) << 14; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 38)) << 20; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 32)) << 26; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 26)) << 32; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 20)) << 38; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 14)) << 44; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 8)) << 50; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 2)) << 56; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 58); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 54)) << 4; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 48)) << 10; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 42)) << 16; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 52]; - tmp |= (src & MASK(uint64_t, 36)) << 22; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 53]; - tmp |= (src & MASK(uint64_t, 30)) << 28; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 54]; - tmp |= (src & MASK(uint64_t, 24)) << 34; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 55]; - tmp |= (src & MASK(uint64_t, 18)) << 40; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 56]; - tmp |= (src & MASK(uint64_t, 12)) << 46; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 57]; - tmp |= (src & MASK(uint64_t, 6)) << 52; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 58); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<59>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 59); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 54)) << 5; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 49)) << 10; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 44)) << 15; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 39)) << 20; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 34)) << 25; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 29)) << 30; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 24)) << 35; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 19)) << 40; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 45); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 14)) << 45; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 9)) << 50; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 55); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 4)) << 55; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 59); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 58)) << 1; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 53)) << 6; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 48)) << 11; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 43)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 38)) << 21; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 33)) << 26; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 28)) << 31; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 23)) << 36; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 18)) << 41; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 13)) << 46; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 51); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 8)) << 51; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 3)) << 56; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 59); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 57)) << 2; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 52)) << 7; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 47)) << 12; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 42)) << 17; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 37)) << 22; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 32)) << 27; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 27)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 22)) << 37; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 17)) << 42; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 47); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 12)) << 47; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 7)) << 52; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 57); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 2)) << 57; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 59); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 56)) << 3; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 51)) << 8; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 46)) << 13; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 41)) << 18; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 36)) << 23; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 31)) << 28; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 26)) << 33; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 21)) << 38; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 43); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 16)) << 43; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 11)) << 48; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 53); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 6)) << 53; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 58); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 1)) << 58; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 59); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 55)) << 4; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 50)) << 9; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 45)) << 14; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 40)) << 19; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 52]; - tmp |= (src & MASK(uint64_t, 35)) << 24; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 53]; - tmp |= (src & MASK(uint64_t, 30)) << 29; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 54]; - tmp |= (src & MASK(uint64_t, 25)) << 34; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 55]; - tmp |= (src & MASK(uint64_t, 20)) << 39; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 56]; - tmp |= (src & MASK(uint64_t, 15)) << 44; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 49); - src = in[lane + LANE_COUNT * 57]; - tmp |= (src & MASK(uint64_t, 10)) << 49; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 54); - src = in[lane + LANE_COUNT * 58]; - tmp |= (src & MASK(uint64_t, 5)) << 54; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 59); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<60>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 60); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 56)) << 4; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 52)) << 8; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 48)) << 12; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 44)) << 16; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 40)) << 20; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 36)) << 24; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 32)) << 28; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 28)) << 32; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 24)) << 36; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 20)) << 40; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 16)) << 44; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 12)) << 48; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 52; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 4)) << 56; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 60); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 0)) << 60; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 60); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 56)) << 4; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 52)) << 8; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 48)) << 12; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 44)) << 16; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 40)) << 20; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 36)) << 24; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 32)) << 28; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 28)) << 32; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 24)) << 36; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 20)) << 40; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 16)) << 44; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 12)) << 48; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 8)) << 52; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 4)) << 56; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 60); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 0)) << 60; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 60); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 56)) << 4; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 52)) << 8; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 48)) << 12; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 44)) << 16; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 40)) << 20; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 36)) << 24; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 32)) << 28; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 28)) << 32; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 24)) << 36; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 20)) << 40; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 16)) << 44; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 12)) << 48; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 8)) << 52; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 4)) << 56; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 60); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 0)) << 60; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 60); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 56)) << 4; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 52)) << 8; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 48)) << 12; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 44)) << 16; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 40)) << 20; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 36)) << 24; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 52]; - tmp |= (src & MASK(uint64_t, 32)) << 28; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 53]; - tmp |= (src & MASK(uint64_t, 28)) << 32; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 54]; - tmp |= (src & MASK(uint64_t, 24)) << 36; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 55]; - tmp |= (src & MASK(uint64_t, 20)) << 40; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 56]; - tmp |= (src & MASK(uint64_t, 16)) << 44; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 57]; - tmp |= (src & MASK(uint64_t, 12)) << 48; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 58]; - tmp |= (src & MASK(uint64_t, 8)) << 52; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 59]; - tmp |= (src & MASK(uint64_t, 4)) << 56; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 60); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<61>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 61); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 58)) << 3; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 55)) << 6; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 52)) << 9; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 49)) << 12; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 46)) << 15; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 43)) << 18; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 40)) << 21; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 37)) << 24; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 34)) << 27; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 31)) << 30; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 28)) << 33; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 25)) << 36; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 22)) << 39; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 19)) << 42; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 45); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 16)) << 45; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 13)) << 48; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 51); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 10)) << 51; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 54); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 7)) << 54; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 57); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 4)) << 57; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 60); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 1)) << 60; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 61); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 59)) << 2; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 56)) << 5; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 53)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 50)) << 11; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 47)) << 14; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 44)) << 17; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 41)) << 20; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 38)) << 23; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 35)) << 26; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 32)) << 29; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 29)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 26)) << 35; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 23)) << 38; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 20)) << 41; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 17)) << 44; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 47); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 14)) << 47; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 11)) << 50; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 53); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 8)) << 53; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 5)) << 56; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 59); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 2)) << 59; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 61); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 60)) << 1; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 57)) << 4; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 54)) << 7; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 51)) << 10; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 48)) << 13; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 45)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 42)) << 19; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 39)) << 22; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 36)) << 25; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 33)) << 28; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 30)) << 31; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 52]; - tmp |= (src & MASK(uint64_t, 27)) << 34; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 53]; - tmp |= (src & MASK(uint64_t, 24)) << 37; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 54]; - tmp |= (src & MASK(uint64_t, 21)) << 40; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 43); - src = in[lane + LANE_COUNT * 55]; - tmp |= (src & MASK(uint64_t, 18)) << 43; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 56]; - tmp |= (src & MASK(uint64_t, 15)) << 46; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 49); - src = in[lane + LANE_COUNT * 57]; - tmp |= (src & MASK(uint64_t, 12)) << 49; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 58]; - tmp |= (src & MASK(uint64_t, 9)) << 52; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 55); - src = in[lane + LANE_COUNT * 59]; - tmp |= (src & MASK(uint64_t, 6)) << 55; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 58); - src = in[lane + LANE_COUNT * 60]; - tmp |= (src & MASK(uint64_t, 3)) << 58; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 61); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<62>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 62); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 60)) << 2; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 58)) << 4; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 56)) << 6; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 54)) << 8; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 52)) << 10; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 50)) << 12; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 48)) << 14; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 46)) << 16; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 44)) << 18; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 42)) << 20; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 40)) << 22; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 38)) << 24; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 36)) << 26; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 34)) << 28; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 32)) << 30; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 30)) << 32; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 28)) << 34; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 26)) << 36; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 24)) << 38; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 22)) << 40; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 20)) << 42; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 18)) << 44; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 16)) << 46; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 14)) << 48; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 12)) << 50; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 10)) << 52; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 54); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 8)) << 54; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 6)) << 56; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 58); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 4)) << 58; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 60); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 2)) << 60; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 62); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 0)) << 62; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 62); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 60)) << 2; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 58)) << 4; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 56)) << 6; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 54)) << 8; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 52)) << 10; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 50)) << 12; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 48)) << 14; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 46)) << 16; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 44)) << 18; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 42)) << 20; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 40)) << 22; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 38)) << 24; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 36)) << 26; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 34)) << 28; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 32)) << 30; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 30)) << 32; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 28)) << 34; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 26)) << 36; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 24)) << 38; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 22)) << 40; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 52]; - tmp |= (src & MASK(uint64_t, 20)) << 42; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 53]; - tmp |= (src & MASK(uint64_t, 18)) << 44; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 54]; - tmp |= (src & MASK(uint64_t, 16)) << 46; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 55]; - tmp |= (src & MASK(uint64_t, 14)) << 48; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 56]; - tmp |= (src & MASK(uint64_t, 12)) << 50; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 57]; - tmp |= (src & MASK(uint64_t, 10)) << 52; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 54); - src = in[lane + LANE_COUNT * 58]; - tmp |= (src & MASK(uint64_t, 8)) << 54; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 59]; - tmp |= (src & MASK(uint64_t, 6)) << 56; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 58); - src = in[lane + LANE_COUNT * 60]; - tmp |= (src & MASK(uint64_t, 4)) << 58; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 60); - src = in[lane + LANE_COUNT * 61]; - tmp |= (src & MASK(uint64_t, 2)) << 60; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 62); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<63>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 63); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 62)) << 1; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 61)) << 2; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 60)) << 3; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 59)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 58)) << 5; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 57)) << 6; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 56)) << 7; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 55)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 54)) << 9; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 53)) << 10; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 52)) << 11; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 51)) << 12; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 50)) << 13; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 49)) << 14; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 48)) << 15; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 47)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 46)) << 17; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 45)) << 18; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 44)) << 19; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 43)) << 20; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 42)) << 21; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 41)) << 22; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 40)) << 23; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 39)) << 24; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 38)) << 25; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 37)) << 26; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 36)) << 27; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 35)) << 28; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 34)) << 29; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 33)) << 30; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 32)) << 31; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 31)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 30)) << 33; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 29)) << 34; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 28)) << 35; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 27)) << 36; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 26)) << 37; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 25)) << 38; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 24)) << 39; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 23)) << 40; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 22)) << 41; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 21)) << 42; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 43); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 20)) << 43; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 19)) << 44; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 45); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 18)) << 45; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 17)) << 46; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 47); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 16)) << 47; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 15)) << 48; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 49); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 14)) << 49; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 13)) << 50; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 51); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 12)) << 51; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 52]; - tmp |= (src & MASK(uint64_t, 11)) << 52; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 53); - src = in[lane + LANE_COUNT * 53]; - tmp |= (src & MASK(uint64_t, 10)) << 53; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 54); - src = in[lane + LANE_COUNT * 54]; - tmp |= (src & MASK(uint64_t, 9)) << 54; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 55); - src = in[lane + LANE_COUNT * 55]; - tmp |= (src & MASK(uint64_t, 8)) << 55; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 56]; - tmp |= (src & MASK(uint64_t, 7)) << 56; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 57); - src = in[lane + LANE_COUNT * 57]; - tmp |= (src & MASK(uint64_t, 6)) << 57; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 58); - src = in[lane + LANE_COUNT * 58]; - tmp |= (src & MASK(uint64_t, 5)) << 58; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 59); - src = in[lane + LANE_COUNT * 59]; - tmp |= (src & MASK(uint64_t, 4)) << 59; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 60); - src = in[lane + LANE_COUNT * 60]; - tmp |= (src & MASK(uint64_t, 3)) << 60; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 61); - src = in[lane + LANE_COUNT * 61]; - tmp |= (src & MASK(uint64_t, 2)) << 61; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 62); - src = in[lane + LANE_COUNT * 62]; - tmp |= (src & MASK(uint64_t, 1)) << 62; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 63); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<64>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - #pragma unroll - for (int row = 0; row < 64; row++) { - out[INDEX(row, lane)] = in[LANE_COUNT * row + lane] + reference; - } -} - -/// Runtime dispatch to the optimized lane decoder for the given bit width. -__device__ __noinline__ void bit_unpack_64_lane( - const uint64_t *__restrict in, - uint64_t *__restrict out, - uint64_t reference, - unsigned int lane, - uint32_t bit_width -) { - switch (bit_width) { - case 0: _bit_unpack_64_lane<0>(in, out, reference, lane); break; - case 1: _bit_unpack_64_lane<1>(in, out, reference, lane); break; - case 2: _bit_unpack_64_lane<2>(in, out, reference, lane); break; - case 3: _bit_unpack_64_lane<3>(in, out, reference, lane); break; - case 4: _bit_unpack_64_lane<4>(in, out, reference, lane); break; - case 5: _bit_unpack_64_lane<5>(in, out, reference, lane); break; - case 6: _bit_unpack_64_lane<6>(in, out, reference, lane); break; - case 7: _bit_unpack_64_lane<7>(in, out, reference, lane); break; - case 8: _bit_unpack_64_lane<8>(in, out, reference, lane); break; - case 9: _bit_unpack_64_lane<9>(in, out, reference, lane); break; - case 10: _bit_unpack_64_lane<10>(in, out, reference, lane); break; - case 11: _bit_unpack_64_lane<11>(in, out, reference, lane); break; - case 12: _bit_unpack_64_lane<12>(in, out, reference, lane); break; - case 13: _bit_unpack_64_lane<13>(in, out, reference, lane); break; - case 14: _bit_unpack_64_lane<14>(in, out, reference, lane); break; - case 15: _bit_unpack_64_lane<15>(in, out, reference, lane); break; - case 16: _bit_unpack_64_lane<16>(in, out, reference, lane); break; - case 17: _bit_unpack_64_lane<17>(in, out, reference, lane); break; - case 18: _bit_unpack_64_lane<18>(in, out, reference, lane); break; - case 19: _bit_unpack_64_lane<19>(in, out, reference, lane); break; - case 20: _bit_unpack_64_lane<20>(in, out, reference, lane); break; - case 21: _bit_unpack_64_lane<21>(in, out, reference, lane); break; - case 22: _bit_unpack_64_lane<22>(in, out, reference, lane); break; - case 23: _bit_unpack_64_lane<23>(in, out, reference, lane); break; - case 24: _bit_unpack_64_lane<24>(in, out, reference, lane); break; - case 25: _bit_unpack_64_lane<25>(in, out, reference, lane); break; - case 26: _bit_unpack_64_lane<26>(in, out, reference, lane); break; - case 27: _bit_unpack_64_lane<27>(in, out, reference, lane); break; - case 28: _bit_unpack_64_lane<28>(in, out, reference, lane); break; - case 29: _bit_unpack_64_lane<29>(in, out, reference, lane); break; - case 30: _bit_unpack_64_lane<30>(in, out, reference, lane); break; - case 31: _bit_unpack_64_lane<31>(in, out, reference, lane); break; - case 32: _bit_unpack_64_lane<32>(in, out, reference, lane); break; - case 33: _bit_unpack_64_lane<33>(in, out, reference, lane); break; - case 34: _bit_unpack_64_lane<34>(in, out, reference, lane); break; - case 35: _bit_unpack_64_lane<35>(in, out, reference, lane); break; - case 36: _bit_unpack_64_lane<36>(in, out, reference, lane); break; - case 37: _bit_unpack_64_lane<37>(in, out, reference, lane); break; - case 38: _bit_unpack_64_lane<38>(in, out, reference, lane); break; - case 39: _bit_unpack_64_lane<39>(in, out, reference, lane); break; - case 40: _bit_unpack_64_lane<40>(in, out, reference, lane); break; - case 41: _bit_unpack_64_lane<41>(in, out, reference, lane); break; - case 42: _bit_unpack_64_lane<42>(in, out, reference, lane); break; - case 43: _bit_unpack_64_lane<43>(in, out, reference, lane); break; - case 44: _bit_unpack_64_lane<44>(in, out, reference, lane); break; - case 45: _bit_unpack_64_lane<45>(in, out, reference, lane); break; - case 46: _bit_unpack_64_lane<46>(in, out, reference, lane); break; - case 47: _bit_unpack_64_lane<47>(in, out, reference, lane); break; - case 48: _bit_unpack_64_lane<48>(in, out, reference, lane); break; - case 49: _bit_unpack_64_lane<49>(in, out, reference, lane); break; - case 50: _bit_unpack_64_lane<50>(in, out, reference, lane); break; - case 51: _bit_unpack_64_lane<51>(in, out, reference, lane); break; - case 52: _bit_unpack_64_lane<52>(in, out, reference, lane); break; - case 53: _bit_unpack_64_lane<53>(in, out, reference, lane); break; - case 54: _bit_unpack_64_lane<54>(in, out, reference, lane); break; - case 55: _bit_unpack_64_lane<55>(in, out, reference, lane); break; - case 56: _bit_unpack_64_lane<56>(in, out, reference, lane); break; - case 57: _bit_unpack_64_lane<57>(in, out, reference, lane); break; - case 58: _bit_unpack_64_lane<58>(in, out, reference, lane); break; - case 59: _bit_unpack_64_lane<59>(in, out, reference, lane); break; - case 60: _bit_unpack_64_lane<60>(in, out, reference, lane); break; - case 61: _bit_unpack_64_lane<61>(in, out, reference, lane); break; - case 62: _bit_unpack_64_lane<62>(in, out, reference, lane); break; - case 63: _bit_unpack_64_lane<63>(in, out, reference, lane); break; - case 64: _bit_unpack_64_lane<64>(in, out, reference, lane); break; - } -} - diff --git a/vortex-cuda/kernels/src/bit_unpack_8.cu b/vortex-cuda/kernels/src/bit_unpack_8.cu deleted file mode 100644 index b2fcfd26f04..00000000000 --- a/vortex-cuda/kernels/src/bit_unpack_8.cu +++ /dev/null @@ -1,95 +0,0 @@ -// AUTO-GENERATED. Do not edit by hand! -#include "bit_unpack_8_lanes.cuh" -#include "patches.cuh" - -template -__device__ void _bit_unpack_8_device(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, int thread_idx, GPUPatches& patches) { - __shared__ uint8_t shared_out[FL_CHUNK]; - - // Step 1: Unpack into shared memory - #pragma unroll - for (int i = 0; i < FL_LANES / 32; i++) { - _bit_unpack_8_lane(in, shared_out, reference, thread_idx * (FL_LANES / 32) + i); - } - __syncwarp(); - - // Step 2: Apply patches to shared memory in parallel - PatchesCursor cursor(patches, blockIdx.x, thread_idx, 32); - auto patch = cursor.next(); - while (patch.index != FL_CHUNK) { - shared_out[patch.index] = patch.value; - patch = cursor.next(); - } - __syncwarp(); - - // Step 3: Copy to global memory - #pragma unroll - for (int i = 0; i < FL_CHUNK / 32; i++) { - auto idx = i * 32 + thread_idx; - out[idx] = shared_out[idx]; - } -} - -extern "C" __global__ void bit_unpack_8_0bw_32t(const uint8_t *__restrict full_in, uint8_t *__restrict full_out, uint8_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 0)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_8_device<0>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_8_1bw_32t(const uint8_t *__restrict full_in, uint8_t *__restrict full_out, uint8_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 1)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_8_device<1>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_8_2bw_32t(const uint8_t *__restrict full_in, uint8_t *__restrict full_out, uint8_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 2)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_8_device<2>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_8_3bw_32t(const uint8_t *__restrict full_in, uint8_t *__restrict full_out, uint8_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 3)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_8_device<3>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_8_4bw_32t(const uint8_t *__restrict full_in, uint8_t *__restrict full_out, uint8_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 4)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_8_device<4>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_8_5bw_32t(const uint8_t *__restrict full_in, uint8_t *__restrict full_out, uint8_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 5)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_8_device<5>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_8_6bw_32t(const uint8_t *__restrict full_in, uint8_t *__restrict full_out, uint8_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 6)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_8_device<6>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_8_7bw_32t(const uint8_t *__restrict full_in, uint8_t *__restrict full_out, uint8_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 7)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_8_device<7>(in, out, reference, thread_idx, patches); -} - -extern "C" __global__ void bit_unpack_8_8bw_32t(const uint8_t *__restrict full_in, uint8_t *__restrict full_out, uint8_t reference, GPUPatches patches) { - int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (FL_LANES * 8)); - auto out = full_out + (blockIdx.x * FL_CHUNK); - _bit_unpack_8_device<8>(in, out, reference, thread_idx, patches); -} - diff --git a/vortex-cuda/kernels/src/bit_unpack_8_lanes.cuh b/vortex-cuda/kernels/src/bit_unpack_8_lanes.cuh deleted file mode 100644 index d9c3550cb18..00000000000 --- a/vortex-cuda/kernels/src/bit_unpack_8_lanes.cuh +++ /dev/null @@ -1,259 +0,0 @@ -// AUTO-GENERATED. Do not edit by hand! -#pragma once - -#include -#include -#include -#include "fastlanes_common.cuh" - -template -__device__ void _bit_unpack_8_lane(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane); - -template <> -__device__ void _bit_unpack_8_lane<0>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - #pragma unroll - for (int row = 0; row < 8; row++) { - out[INDEX(row, lane)] = reference; - } -} - -template <> -__device__ void _bit_unpack_8_lane<1>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint8_t src; - uint8_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint8_t, 1); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint8_t, 1); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint8_t, 1); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint8_t, 1); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 1); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint8_t, 1); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint8_t, 1); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint8_t, 1); - out[INDEX(7, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_8_lane<2>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint8_t src; - uint8_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint8_t, 2); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint8_t, 2); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 2); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint8_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint8_t, 0)) << 2; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint8_t, 2); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint8_t, 2); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 2); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint8_t, 2); - out[INDEX(7, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_8_lane<3>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint8_t src; - uint8_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint8_t, 3); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint8_t, 3); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint8_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint8_t, 1)) << 2; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint8_t, 3); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 3); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint8_t, 1); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint8_t, 2)) << 1; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint8_t, 3); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint8_t, 3); - out[INDEX(7, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_8_lane<4>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint8_t src; - uint8_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint8_t, 4); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint8_t, 0)) << 4; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint8_t, 4); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint8_t, 0)) << 4; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint8_t, 4); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 4); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint8_t, 0)) << 4; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint8_t, 4); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 4); - out[INDEX(7, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_8_lane<5>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint8_t src; - uint8_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint8_t, 5); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint8_t, 3); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint8_t, 2)) << 3; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint8_t, 5); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint8_t, 1); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint8_t, 4)) << 1; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 4); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint8_t, 1)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint8_t, 5); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint8_t, 2); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint8_t, 3)) << 2; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint8_t, 5); - out[INDEX(7, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_8_lane<6>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint8_t src; - uint8_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint8_t, 6); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint8_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint8_t, 4)) << 2; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint8_t, 2)) << 4; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint8_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint8_t, 0)) << 6; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint8_t, 6); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint8_t, 2); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint8_t, 4)) << 2; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 4); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint8_t, 2)) << 4; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint8_t, 6); - out[INDEX(7, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_8_lane<7>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - uint8_t src; - uint8_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint8_t, 7); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint8_t, 1); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint8_t, 6)) << 1; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint8_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint8_t, 5)) << 2; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint8_t, 3); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint8_t, 4)) << 3; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint8_t, 3)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint8_t, 5); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint8_t, 2)) << 5; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint8_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint8_t, 1)) << 6; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint8_t, 7); - out[INDEX(7, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_8_lane<8>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - constexpr unsigned int LANE_COUNT = FL_LANES; - #pragma unroll - for (int row = 0; row < 8; row++) { - out[INDEX(row, lane)] = in[LANE_COUNT * row + lane] + reference; - } -} - -/// Runtime dispatch to the optimized lane decoder for the given bit width. -__device__ __noinline__ void bit_unpack_8_lane( - const uint8_t *__restrict in, - uint8_t *__restrict out, - uint8_t reference, - unsigned int lane, - uint32_t bit_width -) { - switch (bit_width) { - case 0: _bit_unpack_8_lane<0>(in, out, reference, lane); break; - case 1: _bit_unpack_8_lane<1>(in, out, reference, lane); break; - case 2: _bit_unpack_8_lane<2>(in, out, reference, lane); break; - case 3: _bit_unpack_8_lane<3>(in, out, reference, lane); break; - case 4: _bit_unpack_8_lane<4>(in, out, reference, lane); break; - case 5: _bit_unpack_8_lane<5>(in, out, reference, lane); break; - case 6: _bit_unpack_8_lane<6>(in, out, reference, lane); break; - case 7: _bit_unpack_8_lane<7>(in, out, reference, lane); break; - case 8: _bit_unpack_8_lane<8>(in, out, reference, lane); break; - } -} - diff --git a/vortex-cuda/nvcomp/build.rs b/vortex-cuda/nvcomp/build.rs index 877fc68c58d..66ef92618b6 100644 --- a/vortex-cuda/nvcomp/build.rs +++ b/vortex-cuda/nvcomp/build.rs @@ -1,10 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! Downloads the nvCOMP SDK and generates Rust FFI bindings for nvCOMP Zstd. +//! Downloads the nvCOMP SDK and generates Rust FFI bindings for nvCOMP Zstd on +//! supported targets. //! -//! Bindings are generated unconditionally. This allows for development against the -//! CUDA APIs in environments that don't support CUDA. +//! On unsupported targets, the build emits stub bindings instead. This allows +//! development against the CUDA APIs in environments that don't support CUDA +//! without downloading Linux-only NVIDIA artifacts. //! //! The library is loaded at runtime via libloading. @@ -33,22 +35,22 @@ typedef int cudaError_t; fn main() { println!("cargo:rerun-if-env-changed=CUDA_PATH"); + println!("cargo:rustc-check-cfg=cfg(nvcomp_stub)"); let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); - let nvcomp_dir = out_dir.join("nvcomp-sdk"); + + let Some((os, arch)) = nvcomp_archive_target() else { + write_stub_bindings(&out_dir); + println!("cargo:rustc-cfg=nvcomp_stub"); + return; + }; // Create CUDA stub header in OUT_DIR for bindgen let cuda_stub_dir = out_dir.join("cuda-stub"); fs::create_dir_all(&cuda_stub_dir).unwrap(); fs::write(cuda_stub_dir.join("cuda_runtime.h"), CUDA_RUNTIME_STUB).unwrap(); - let (os, arch) = match (env::consts::OS, env::consts::ARCH) { - ("linux", "x86_64") => ("linux", "x86_64"), - ("linux", "aarch64") => ("linux", "sbsa"), - // Fall back to linux-x86_64 to generate bindings for any platform. - _ => ("linux", "x86_64"), - }; - + let nvcomp_dir = out_dir.join("nvcomp-sdk"); let archive_name = format!("nvcomp-{os}-{arch}-{NVCOMP_VERSION}_{CUDA_VERSION}-archive"); let url = format!( "https://developer.download.nvidia.com/compute/nvcomp/redist/nvcomp/{os}-{arch}/{archive_name}.tar.xz" @@ -111,3 +113,97 @@ fn main() { bindings.write_to_file(out_dir.join("sys.rs")).unwrap(); } + +fn nvcomp_archive_target() -> Option<(&'static str, &'static str)> { + let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap(); + let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap(); + + match (target_os.as_str(), target_arch.as_str()) { + ("linux", "x86_64") => Some(("linux", "x86_64")), + ("linux", "aarch64") => Some(("linux", "sbsa")), + _ => None, + } +} + +fn write_stub_bindings(out_dir: &std::path::Path) { + fs::write( + out_dir.join("sys.rs"), + r#" +// Stub FFI type definitions for targets without nvCOMP SDK support. + +pub type cudaStream_t = *mut std::ffi::c_void; + +pub const nvcompStatus_t_nvcompSuccess: nvcompStatus_t = 0; +pub const nvcompStatus_t_nvcompErrorInvalidValue: nvcompStatus_t = 10; +pub const nvcompStatus_t_nvcompErrorNotSupported: nvcompStatus_t = 11; +pub const nvcompStatus_t_nvcompErrorCannotDecompress: nvcompStatus_t = 12; +pub const nvcompStatus_t_nvcompErrorBadChecksum: nvcompStatus_t = 13; +pub const nvcompStatus_t_nvcompErrorCannotVerifyChecksums: nvcompStatus_t = 14; +pub const nvcompStatus_t_nvcompErrorOutputBufferTooSmall: nvcompStatus_t = 15; +pub const nvcompStatus_t_nvcompErrorWrongHeaderLength: nvcompStatus_t = 16; +pub const nvcompStatus_t_nvcompErrorAlignment: nvcompStatus_t = 17; +pub const nvcompStatus_t_nvcompErrorChunkSizeTooLarge: nvcompStatus_t = 18; +pub const nvcompStatus_t_nvcompErrorCannotCompress: nvcompStatus_t = 19; +pub const nvcompStatus_t_nvcompErrorWrongInputLength: nvcompStatus_t = 20; +pub const nvcompStatus_t_nvcompErrorBatchSizeTooLarge: nvcompStatus_t = 21; +pub const nvcompStatus_t_nvcompErrorCudaError: nvcompStatus_t = 1000; +pub const nvcompStatus_t_nvcompErrorInternal: nvcompStatus_t = 10000; +pub type nvcompStatus_t = ::std::os::raw::c_uint; + +pub const nvcompDecompressBackend_t_NVCOMP_DECOMPRESS_BACKEND_DEFAULT: nvcompDecompressBackend_t = + 0; +pub const nvcompDecompressBackend_t_NVCOMP_DECOMPRESS_BACKEND_HARDWARE: nvcompDecompressBackend_t = + 1; +pub const nvcompDecompressBackend_t_NVCOMP_DECOMPRESS_BACKEND_CUDA: nvcompDecompressBackend_t = 2; +pub type nvcompDecompressBackend_t = ::std::os::raw::c_uint; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct nvcompBatchedZstdDecompressOpts_t { + pub backend: nvcompDecompressBackend_t, + pub reserved: [::std::os::raw::c_char; 60usize], +} + +pub struct NvcompLibrary; + +impl NvcompLibrary { + pub unsafe fn new

(_path: P) -> Result + where + P: AsRef<::std::ffi::OsStr>, + { + Ok(Self) + } + + pub unsafe fn nvcompBatchedZstdDecompressGetTempSizeAsync( + &self, + _num_chunks: usize, + _max_uncompressed_chunk_bytes: usize, + _decompress_opts: nvcompBatchedZstdDecompressOpts_t, + _temp_bytes: *mut usize, + _max_total_uncompressed_bytes: usize, + ) -> nvcompStatus_t { + nvcompStatus_t_nvcompErrorNotSupported + } + + #[expect(clippy::too_many_arguments)] + pub unsafe fn nvcompBatchedZstdDecompressAsync( + &self, + _device_compressed_chunk_ptrs: *const *const ::std::os::raw::c_void, + _device_compressed_chunk_bytes: *const usize, + _device_uncompressed_buffer_bytes: *const usize, + _device_uncompressed_chunk_bytes: *mut usize, + _num_chunks: usize, + _device_temp_ptr: *mut ::std::os::raw::c_void, + _temp_bytes: usize, + _device_uncompressed_chunk_ptrs: *const *mut ::std::os::raw::c_void, + _decompress_opts: nvcompBatchedZstdDecompressOpts_t, + _device_statuses: *mut nvcompStatus_t, + _stream: cudaStream_t, + ) -> nvcompStatus_t { + nvcompStatus_t_nvcompErrorNotSupported + } +} +"#, + ) + .unwrap(); +} diff --git a/vortex-cuda/nvcomp/src/lib.rs b/vortex-cuda/nvcomp/src/lib.rs index 3ab502baa63..e5b9fe7421f 100644 --- a/vortex-cuda/nvcomp/src/lib.rs +++ b/vortex-cuda/nvcomp/src/lib.rs @@ -4,20 +4,21 @@ //! Rust bindings to NVIDIA nvCOMP compression library. //! //! This crate provides bindings to nvCOMP, with the library loaded at runtime -//! via `libloading`. This allows the crate to compile on systems without CUDA -//! or nvcomp installed - the library is only required at runtime when the +//! via `libloading`. This allows the crate to compile on Linux systems without +//! CUDA or nvcomp installed - the library is only required at runtime when the //! functions are actually called. //! //! # Platform Support //! //! nvCOMP is only available on Linux x86_64 and ARM64. On other platforms, -//! this crate still compiles but will fail at runtime when trying to load -//! the library. +//! this crate emits stub bindings without downloading the SDK and returns a +//! runtime error when trying to load the library. //! //! # Runtime Requirements //! //! The nvcomp library must be available at runtime. +#[cfg(not(nvcomp_stub))] use std::path::PathBuf; use std::sync::OnceLock; @@ -38,6 +39,7 @@ pub use error::NvcompError; /// The loaded nvcomp library instance. static NVCOMP_LIB: OnceLock> = OnceLock::new(); +#[cfg(not(nvcomp_stub))] fn load_nvcomp() -> Result { let lib_name = "libnvcomp.so"; let build_lib_dir = env!("OUT_DIR"); @@ -47,13 +49,18 @@ fn load_nvcomp() -> Result { .join(lib_name); // SAFETY: The library at the SDK path is a valid nvcomp shared library - // downloaded during the build process. + // downloaded during the build process on supported targets. unsafe { sys::NvcompLibrary::new(&sdk_lib_path) .map_err(|e| format!("Failed to load nvcomp library: {e}")) } } +#[cfg(nvcomp_stub)] +fn load_nvcomp() -> Result { + Err("nvCOMP is only available for Linux x86_64 and aarch64 targets".to_string()) +} + /// Gets a reference to the loaded nvcomp library. /// /// The library is loaded lazily on first access. Returns an error if the From 06c7e5858dd4c5be5954d02913ca82de9c15b791 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 19 Jun 2026 14:49:41 +0100 Subject: [PATCH 2/2] fix CI Signed-off-by: Adam Gutglick --- .github/workflows/codspeed.yml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 6b3af685e7b..8d5d64dbaee 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -100,10 +100,23 @@ jobs: --bench fsst_cuda \ --bench runend_cuda \ --profile bench - - name: Verify CUDA kernels were built - run: test -f vortex-cuda/kernels/gen/release/dynamic_dispatch.ptx + - name: Locate built CUDA kernels + id: cuda_kernels + run: | + ptx_file="$(find target -type f -path '*/out/kernels/*/dynamic_dispatch.ptx' -print -quit)" + if [ -z "$ptx_file" ]; then + echo "::error::dynamic_dispatch.ptx not found under target/**/out/kernels/*" + find target -type d -path '*/out/kernels*' -print + exit 1 + fi + + kernels_dir="$(dirname "$ptx_file")" + echo "kernels_dir=$kernels_dir" >> "$GITHUB_OUTPUT" + find "$kernels_dir" -maxdepth 1 -name '*.ptx' -print | sort - name: Stage CUDA kernels with Codspeed artifacts - run: cp -R vortex-cuda/kernels/gen/release target/codspeed/cuda-kernels + run: | + mkdir -p target/codspeed/cuda-kernels + cp -R "${{ steps.cuda_kernels.outputs.kernels_dir }}/." target/codspeed/cuda-kernels - name: Upload benchmark executables uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: