use standard bitrank instead of virtualized version

add quaternary parallel implementation (~50ns vs. ~80ns for binary).

Author: aneubeck

crates/quaternary_trie/src/lib.rs

@@ -1,6 +1,7 @@
 use virtual_bitrank::{VirtualBitRank, Word, WORD_BITS};
 
 pub mod parallel;
+mod parallel4;
 mod virtual_bitrank;
 
 const MAX_LEVEL: usize = 14;
@@ -200,22 +201,13 @@ impl QuarternaryTrie {
             s.fill_bit_rank(&mut consumed, MAX_LEVEL - 1);
         }
         s.data.build();
-        s.reset_stats();
         println!(
             "encoded size: {}",
             4.0 * s.level_idx[0] as f32 / values.len() as f32
         );
         s
     }
 
-    fn reset_stats(&mut self) {
-        self.data.reset_stats();
-    }
-
-    fn page_count(&self) -> (usize, usize) {
-        self.data.page_count()
-    }
-
     fn recurse(&self, node: usize, level: usize, value: u32, results: &mut Vec<u32>) {
         if level == 1 {
             self.recurse2(node, value, results);
@@ -567,10 +559,10 @@ mod tests {
 
     #[test]
     fn test_large() {
-        let mut values: Vec<_> = (0..10000000)
+        let mut values: Vec<_> = (0..100_000)
             .map(|_| thread_rng().gen_range(0..100000000))
             .collect();
-        // let mut values: Vec<_> = (0..100).map(|_| thread_rng().gen_range(0..10000)).collect();
+        // let mut values: Vec<_> = (0..10).map(|_| thread_rng().gen_range(0..100)).collect();
         values.sort();
         values.dedup();
 
@@ -587,7 +579,7 @@ mod tests {
         let start = Instant::now();
         let result: Vec<_> = iter.collect();
         println!("iteration {:?}", start.elapsed() / values.len() as u32);
-        // assert_eq!(result, values);
+        assert_eq!(result, values);
     }
 
     #[test]
@@ -670,17 +662,12 @@ mod tests {
                 ));
                 let start = Instant::now();
                 let result: Vec<_> = iter.collect();
-                let count = trie.page_count();
-                let count2 = trie2.page_count();
-                page_counts[i] += count.0 + count2.0;
                 println!(
-                    "trie intersection {:?} {}",
+                    "trie intersection {:?}",
                     start.elapsed() / values.len() as u32,
-                    (count.0 + count2.0) as f32 / (count.1 + count2.1) as f32
                 );
                 assert_eq!(result, intersection);
             }
-            println!("{page_counts:?}");
         }
     }
 

crates/quaternary_trie/src/parallel.rs

@@ -1,4 +1,4 @@
-use std::arch::x86_64::{_pdep_u64, _pext_u64};
+use std::arch::x86_64::_pdep_u64;
 
 use crate::virtual_bitrank::VirtualBitRank;
 
@@ -21,10 +21,10 @@ impl ParallelTrie {
         // !("fill_bit_rank {prefix} {mask:064b} {level}");
         for t in [0, 64 << level] {
             let mut sub_mask = 0;
-            for i in 0..64 {
-                if (1 << i) & mask == 0 {
-                    continue;
-                }
+            let mut mask = mask;
+            while mask != 0 {
+                let i = mask.trailing_zeros() as usize;
+                mask &= mask - 1;
                 if let Some(&value) = slices[i].get(0) {
                     if (value ^ prefix) >> (level + 7) == 0 && value & (64 << level) == t {
                         if WRITE {
@@ -134,19 +134,46 @@ impl ParallelTrie {
             }
         } else {
             let required_bits = word.count_ones();
-            if required_bits == 0 {
-                return;
+            let mut new_rank = self.data.rank(rank) as usize + self.root_ones;
+
+            if required_bits == 1 {
+                // TODO: simply switch to single bit recursion here instead of checking on every level again.
+                // NOTE: we cannot easily read here a nibble, since the rank is only a multiple of 2, but not necessarily of 4.
+                let mut w = self.data.get_word(rank) & 3;
+                while w != 0 {
+                    let zeros = w.trailing_zeros();
+                    w &= w - 1;
+                    self.recurse(pos * 2 + zeros as usize, word, new_rank * 2, level - 1, v);
+                    new_rank += 1;
+                }
+            } else if required_bits <= 32 {
+                let w = self.data.get_word(rank);
+                let new_word = unsafe { _pdep_u64(w, word) };
+                if new_word != 0 {
+                    self.recurse(pos * 2, new_word, new_rank * 2, level - 1, v);
+                    new_rank += new_word.count_ones() as usize;
+                }
+
+                let w = w >> required_bits;
+                let new_word = unsafe { _pdep_u64(w, word) };
+                if new_word != 0 {
+                    self.recurse(pos * 2 + 1, new_word, new_rank * 2, level - 1, v);
+                }
+            } else {
+                let w = self.data.get_word(rank);
+                let new_word = unsafe { _pdep_u64(w, word) };
+                if new_word != 0 {
+                    self.recurse(pos * 2, new_word, new_rank * 2, level - 1, v);
+                    new_rank += new_word.count_ones() as usize;
+                }
+
+                let rank = rank + required_bits as usize;
+                let w = self.data.get_word(rank);
+                let new_word = unsafe { _pdep_u64(w, word) };
+                if new_word != 0 {
+                    self.recurse(pos * 2 + 1, new_word, new_rank * 2, level - 1, v);
+                }
             }
-            let w = self.data.get_word(rank);
-            let new_word = unsafe { _pdep_u64(w, word) };
-            let new_rank = self.data.rank(rank) as usize + self.root_ones;
-            self.recurse(pos * 2, new_word, new_rank * 2, level - 1, v);
-
-            let rank = rank + required_bits as usize;
-            let w = self.data.get_word(rank);
-            let new_word = unsafe { _pdep_u64(w, word) };
-            let new_rank = self.data.rank(rank) as usize + self.root_ones;
-            self.recurse(pos * 2 + 1, new_word, new_rank * 2, level - 1, v);
         }
     }
 }
@@ -155,19 +182,15 @@ impl ParallelTrie {
 mod tests {
     use std::time::Instant;
 
-    use itertools::{kmerge, Itertools};
     use rand::{thread_rng, Rng};
 
-    use crate::{
-        parallel::ParallelTrie, Intersection, Layout, QuarternaryTrie, TrieIterator, TrieTraversal,
-        Union,
-    };
+    use crate::parallel::ParallelTrie;
 
     #[test]
-    fn test_parallel() {
+    fn test_parallel_large() {
         // let values = vec![3, 6, 7, 10, 90, 91, 120, 128, 129, 130, 231, 321, 999];
         // let values = vec![3, 6, 7, 321, 999];
-        let mut values: Vec<_> = (0..10_000_000)
+        let mut values: Vec<_> = (0..100_000)
             .map(|_| thread_rng().gen_range(0..100_000_000))
             .collect();
         values.sort();